Skip to content

Commit

Permalink
storage: add benchwrapper, support STORAGE_EMULATOR_HOST
Browse files Browse the repository at this point in the history
  • Loading branch information
jeanbza committed Sep 12, 2019
1 parent cd81f4e commit b427c2b
Show file tree
Hide file tree
Showing 8 changed files with 383 additions and 6 deletions.
3 changes: 3 additions & 0 deletions core/google/cloud/environment_vars.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
BIGTABLE_EMULATOR = "BIGTABLE_EMULATOR_HOST"
"""Environment variable defining host for Bigtable emulator."""

STORAGE_EMULATOR = "STORAGE_EMULATOR_HOST"
"""Environment variable defining host for Storage emulator."""

DISABLE_GRPC = "GOOGLE_CLOUD_DISABLE_GRPC"
"""Environment variable acting as flag to disable gRPC.
Expand Down
11 changes: 5 additions & 6 deletions storage/google/cloud/storage/blob.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,14 @@
from google.cloud.storage.acl import ACL
from google.cloud.storage.acl import ObjectACL

_DEFAULT_STORAGE_HOST = u"https://www.googleapis.com"
_STORAGE_HOST = os.environ.get(
"STORAGE_EMULATOR_HOST", _DEFAULT_STORAGE_HOST)

_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
_DEFAULT_CONTENT_TYPE = u"application/octet-stream"
_DOWNLOAD_URL_TEMPLATE = (
u"https://www.googleapis.com/download/storage/v1{path}?alt=media"
)
_BASE_UPLOAD_TEMPLATE = (
u"https://www.googleapis.com/upload/storage/v1{bucket_path}/o?uploadType="
)
_DOWNLOAD_URL_TEMPLATE = _STORAGE_HOST + u"/download/storage/v1{path}?alt=media"
_BASE_UPLOAD_TEMPLATE = _STORAGE_HOST + u"/upload/storage/v1{bucket_path}/o?uploadType="
_MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"multipart"
_RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + u"resumable"
# NOTE: "acl" is also writeable but we defer ACL management to
Expand Down
6 changes: 6 additions & 0 deletions storage/google/cloud/storage/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@

"""Client for interacting with the Google Cloud Storage API."""

import os

import google.api_core.client_options

from google.auth.credentials import AnonymousCredentials
Expand Down Expand Up @@ -94,6 +96,10 @@ def __init__(
)

kw_args = {"client_info": client_info}

if os.environ.get("STORAGE_EMULATOR_HOST"):
kw_args["api_endpoint"] = os.environ.get("STORAGE_EMULATOR_HOST")

if client_options:
if type(client_options) == dict:
client_options = google.api_core.client_options.from_dict(
Expand Down
20 changes: 20 additions & 0 deletions storage/test_utils/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# storage benchwrapp

main.py is a gRPC wrapper around the storage library for benchmarking purposes.

## Running

```
pip install grpcio
cd storage
pip install -e . # install google.cloud.storage locally
cd test_utils
python3 benchwrapper.py --port 8081
```

## Re-generating protos

```
pip install grpcio-tools
python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. *.proto
```
48 changes: 48 additions & 0 deletions storage/test_utils/benchwrapper.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import argparse
import sys
import time
import grpc
import os
from concurrent import futures
import storage_pb2_grpc
import storage_pb2
from google.cloud import storage

_ONE_DAY_IN_SECONDS = 60 * 60 * 24

parser = argparse.ArgumentParser()

if os.environ.get('STORAGE_EMULATOR_HOST') == None:
sys.exit('This benchmarking server only works when connected to an emulator. Please set STORAGE_EMULATOR_HOST.')

parser.add_argument('--port', help='The port to run on.')

args = parser.parse_args()

if args.port == None:
sys.exit('Usage: python3 main.py --port 8081')

client = storage.Client()

class StorageBenchWrapperServicer(storage_pb2_grpc.StorageBenchWrapperServicer):
def Write(self, request, context):
# TODO(deklerk): implement this
return storage_pb2.EmptyResponse()

def Read(self, request, context):
bucket = client.bucket(request.bucketName)
blob = storage.Blob(request.objectName, bucket)
blob.download_as_string()
return storage_pb2.EmptyResponse()

server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
storage_pb2_grpc.add_StorageBenchWrapperServicer_to_server(StorageBenchWrapperServicer(), server)

print('listening on localhost:'+args.port)
server.add_insecure_port('[::]:'+args.port)
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
43 changes: 43 additions & 0 deletions storage/test_utils/storage.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package storage_bench;

message ObjectRead{
// The bucket string identifier.
string bucketName = 1;
// The object/blob string identifier.
string objectName = 2;
}

message ObjectWrite{
// The bucket string identifier.
string bucketName = 1;
// The object/blob string identifiers.
string objectName = 2;
// The string containing the upload file path.
string destination = 3;
}

message EmptyResponse{
}

service StorageBenchWrapper{
// Performs an upload from a specific object.
rpc Write(ObjectWrite) returns (EmptyResponse) {}
// Read a specific object.
rpc Read(ObjectRead) returns (EmptyResponse){}
}
195 changes: 195 additions & 0 deletions storage/test_utils/storage_pb2.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit b427c2b

Please sign in to comment.