diff --git a/.coveragerc b/.coveragerc index d8f7e700..ce326524 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,4 +3,6 @@ omit = *noseplugin* autopush/tests/certs/makecerts.py autopush/gcdump.py + autopush/webpush_server.py + autopush/tests/test_webpush_server.py show_missing = true diff --git a/.travis.yml b/.travis.yml index bce045b0..604cfad0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ before_install: install: - ${DDB:+make ddb} - pip install tox ${CODECOV:+codecov} +- curl https://sh.rustup.rs | sh -s -- -y +- export PATH=$PATH:$HOME/.cargo/bin script: - tox -- ${CODECOV:+--with-coverage --cover-xml --cover-package=autopush} after_success: diff --git a/Dockerfile b/Dockerfile index a13478c7..b05a380c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,12 @@ RUN mkdir -p /app ADD . /app WORKDIR /app +ENV PATH=$PATH:/root/.cargo/bin RUN \ apt-get update && \ apt-get install -y -qq libexpat1-dev gcc libssl-dev libffi-dev && \ + curl https://sh.rustup.rs | sh -s -- -y && \ make clean && \ pip install -r requirements.txt && \ pypy setup.py develop diff --git a/Dockerfile.python27 b/Dockerfile.python27 index 41eca88b..04112c4a 100644 --- a/Dockerfile.python27 +++ b/Dockerfile.python27 @@ -4,10 +4,12 @@ RUN mkdir -p /app COPY . /app WORKDIR /app +ENV PATH=$PATH:/root/.cargo/bin RUN \ apt-get update && \ apt-get install -y -qq libexpat1-dev gcc libssl-dev libffi-dev && \ + curl https://sh.rustup.rs | sh -s -- -y && \ make clean && \ pip install -r requirements.txt && \ python setup.py develop diff --git a/autopush/main.py b/autopush/main.py index 096a6ce8..8107a5f2 100644 --- a/autopush/main.py +++ b/autopush/main.py @@ -35,6 +35,7 @@ from autopush.logging import PushLogger from autopush.main_argparse import parse_connection, parse_endpoint from autopush.router import routers_from_config +from autopush.webpush_server import WebPushServer from autopush.websocket import ( ConnectionWSSite, PushServerFactory, @@ -287,3 +288,94 @@ def from_argparse(cls, ns): max_connections=ns.max_connections, close_handshake_timeout=ns.close_handshake_timeout, ) + + +class RustConnectionApplication(AutopushMultiService): + """The autopush application""" + + config_files = AutopushMultiService.shared_config_files + ( + '/etc/autopush_connection.ini', + 'configs/autopush_connection.ini', + '~/.autopush_connection.ini', + '.autopush_connection.ini' + ) + + parse_args = staticmethod(parse_connection) # type: ignore + logger_name = "AutopushRust" + push_server = None + + def __init__(self, conf): + # type: (AutopushConfig) -> None + super(RustConnectionApplication, self).__init__(conf) + + def setup(self, rotate_tables=True): + self.db.setup(self.conf.preflight_uaid) + + if self.conf.memusage_port: + self.add_memusage() + + self.push_server = WebPushServer(self.conf, self.db, num_threads=10) + + def run(self): + try: + self.push_server.run() + finally: + self.stopService() + + @inlineCallbacks + def stopService(self): + yield super(RustConnectionApplication, self).stopService() + + @classmethod + def from_argparse(cls, ns): + # type: (Namespace) -> AutopushMultiService + return super(RustConnectionApplication, cls)._from_argparse( + ns, + port=ns.port, + endpoint_scheme=ns.endpoint_scheme, + endpoint_hostname=ns.endpoint_hostname, + endpoint_port=ns.endpoint_port, + router_scheme="https" if ns.router_ssl_key else "http", + router_hostname=ns.router_hostname, + router_port=ns.router_port, + env=ns.env, + hello_timeout=ns.hello_timeout, + router_ssl=dict( + key=ns.router_ssl_key, + cert=ns.router_ssl_cert, + dh_param=ns.ssl_dh_param + ), + # XXX: default is for autopush_rs + auto_ping_interval=ns.auto_ping_interval or 300, + auto_ping_timeout=ns.auto_ping_timeout, + max_connections=ns.max_connections, + close_handshake_timeout=ns.close_handshake_timeout, + ) + + @classmethod + def main(cls, args=None, use_files=True): + # type: (Sequence[str], bool) -> Any + """Entry point to autopush's main command line scripts. + + aka autopush/autoendpoint. + + """ + ns = cls.parse_args(cls.config_files if use_files else [], args) + if not ns.no_aws: + logging.HOSTNAME = utils.get_ec2_instance_id() + PushLogger.setup_logging( + cls.logger_name, + log_level=ns.log_level or ("debug" if ns.debug else "info"), + log_format="text" if ns.human_logs else "json", + log_output=ns.log_output, + sentry_dsn=bool(os.environ.get("SENTRY_DSN")), + firehose_delivery_stream=ns.firehose_stream_name + ) + try: + app = cls.from_argparse(ns) + except InvalidConfig as e: + log.critical(str(e)) + return 1 + + app.setup() + app.run() diff --git a/autopush/tests/test_webpush_server.py b/autopush/tests/test_webpush_server.py new file mode 100644 index 00000000..b33c6da5 --- /dev/null +++ b/autopush/tests/test_webpush_server.py @@ -0,0 +1,545 @@ +import random +import time +import unittest +from threading import Event +from uuid import uuid4, UUID + +import attr +import factory +from boto.dynamodb2.exceptions import ItemNotFound +from boto.dynamodb2.exceptions import ProvisionedThroughputExceededException +from mock import Mock +from nose.tools import assert_raises, ok_, eq_ +from twisted.logger import globalLogPublisher + +from autopush.db import ( + DatabaseManager, + make_rotating_tablename, + generate_last_connect, +) +from autopush.metrics import SinkMetrics +from autopush.config import AutopushConfig +from autopush.logging import begin_or_register +from autopush.tests.support import TestingLogObserver +from autopush.utils import WebPushNotification, ns_time +from autopush.websocket import USER_RECORD_VERSION +from autopush.webpush_server import ( + CheckStorage, + DeleteMessage, + DropUser, + Hello, + HelloResponse, + IncStoragePosition, + MigrateUser, + Register, + StoreMessages, + Unregister, + WebPushMessage, +) + + +class AutopushCall(object): + """Placeholder object for real Rust binding one""" + called = Event() + val = None + payload = None + + def complete(self, ret): + self.val = ret + self.called.set() + + def json(self): + return self.payload + + +class UserItemFactory(factory.Factory): + class Meta: + model = dict + + uaid = factory.LazyFunction(lambda: uuid4().hex) + connected_at = factory.LazyFunction(lambda: int(time.time() * 1000)-10000) + node_id = "http://something:3242/" + router_type = "webpush" + last_connect = factory.LazyFunction(generate_last_connect) + record_version = USER_RECORD_VERSION + current_month = factory.LazyFunction( + lambda: make_rotating_tablename("message") + ) + + +def generate_random_headers(): + return dict( + encryption="aesgcm128", + encryption_key="someneatkey", + crypto_key="anotherneatkey", + ) + + +class WebPushNotificationFactory(factory.Factory): + class Meta: + model = WebPushNotification + + uaid = factory.LazyFunction(uuid4) + channel_id = factory.LazyFunction(uuid4) + ttl = 86400 + data = factory.LazyFunction( + lambda: random.randint(30, 4096) * "*" + ) + headers = factory.LazyFunction(generate_random_headers) + + +def generate_version(obj): + if obj.topic: + msg_key = ":".join(["01", obj.uaid, obj.channelID.hex, + obj.topic]) + else: + sortkey_timestamp = ns_time() + msg_key = ":".join(["02", obj.uaid, obj.channelID.hex, + str(sortkey_timestamp)]) + # Technically this should be fernet encrypted, but this is fine for + # testing here + return msg_key + + +class WebPushMessageFactory(factory.Factory): + class Meta: + model = WebPushMessage + + uaid = factory.LazyFunction(lambda: str(uuid4())) + channelID = factory.LazyFunction(uuid4) + ttl = 86400 + data = factory.LazyFunction( + lambda: random.randint(30, 4096) * "*" + ) + topic = None + timestamp = factory.LazyFunction(lambda: int(time.time() * 1000)) + headers = factory.LazyFunction(generate_random_headers) + version = factory.LazyAttribute(generate_version) + + +class HelloFactory(factory.Factory): + class Meta: + model = Hello + + uaid = factory.LazyFunction(lambda: uuid4().hex) + connected_at = factory.LazyFunction(lambda: int(time.time() * 1000)) + + +class CheckStorageFactory(factory.Factory): + class Meta: + model = CheckStorage + + uaid = factory.LazyFunction(lambda: uuid4().hex) + include_topic = True + + +def webpush_messages(obj): + return [attr.asdict(WebPushMessageFactory(uaid=obj.uaid)) + for _ in range(obj.message_count)] + + +class StoreMessageFactory(factory.Factory): + class Meta: + model = StoreMessages + + messages = factory.LazyAttribute(webpush_messages) + message_month = factory.LazyFunction( + lambda: make_rotating_tablename("message") + ) + + class Params: + message_count = 20 + uaid = factory.LazyFunction(lambda: uuid4().hex) + + +class BaseSetup(unittest.TestCase): + def setUp(self): + self.conf = AutopushConfig( + hostname="localhost", + port=8080, + statsd_host=None, + env="test", + auto_ping_interval=float(300), + auto_ping_timeout=float(10), + close_handshake_timeout=10, + max_connections=2000000, + ) + + self.logs = TestingLogObserver() + begin_or_register(self.logs) + self.addCleanup(globalLogPublisher.removeObserver, self.logs) + + self.db = db = DatabaseManager.from_config(self.conf) + self.metrics = db.metrics = Mock(spec=SinkMetrics) + db.setup_tables() + + def _store_messages(self, uaid, topic=False, num=5): + try: + item = self.db.router.get_uaid(uaid.hex) + message_table = self.db.message_tables[item["current_month"]] + except ItemNotFound: + message_table = self.db.message + messages = [WebPushNotificationFactory(uaid=uaid) + for _ in range(num)] + channels = set([m.channel_id for m in messages]) + for channel in channels: + message_table.register_channel(uaid.hex, channel.hex) + for idx, notif in enumerate(messages): + if topic: + notif.topic = "something_{}".format(idx) + notif.generate_message_id(self.conf.fernet) + message_table.store_message(notif) + return messages + + +class TestWebPushServer(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import WebPushServer + return WebPushServer(self.conf, self.db, num_threads=2) + + def test_start_stop(self): + ws = self._makeFUT() + ws.start() + try: + eq_(len(ws.workers), 2) + finally: + ws.stop() + + def test_hello_process(self): + ws = self._makeFUT() + ws.start() + try: + hello = HelloFactory() + result = ws.command_processor.process_message(dict( + command="hello", + uaid=hello.uaid.hex, + connected_at=hello.connected_at, + )) + ok_("error" not in result) + ok_(hello.uaid.hex != result["uaid"]) + finally: + ws.stop() + + +class TestHelloProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import HelloCommand + return HelloCommand(self.conf, self.db) + + def test_nonexisting_uaid(self): + p = self._makeFUT() + hello = HelloFactory() + result = p.process(hello) # type: HelloResponse + ok_(isinstance(result, HelloResponse)) + ok_(hello.uaid != result.uaid) + + def test_existing_uaid(self): + p = self._makeFUT() + hello = HelloFactory() + success, _ = self.db.router.register_user(UserItemFactory( + uaid=hello.uaid.hex)) + eq_(success, True) + result = p.process(hello) # type: HelloResponse + ok_(isinstance(result, HelloResponse)) + eq_(hello.uaid.hex, result.uaid) + + def test_existing_newer_uaid(self): + p = self._makeFUT() + hello = HelloFactory() + self.db.router.register_user( + UserItemFactory(uaid=hello.uaid.hex, + connected_at=hello.connected_at+10) + ) + result = p.process(hello) # type: HelloResponse + ok_(isinstance(result, HelloResponse)) + eq_(result.uaid, None) + + +class TestCheckStorageProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import CheckStorageCommand + return CheckStorageCommand(self.conf, self.db) + + def test_no_messages(self): + p = self._makeFUT() + check = CheckStorageFactory(message_month=self.db.current_msg_month) + result = p.process(check) + eq_(len(result.messages), 0) + + def test_five_messages(self): + p = self._makeFUT() + check = CheckStorageFactory(message_month=self.db.current_msg_month) + self._store_messages(check.uaid, num=5) + result = p.process(check) + eq_(len(result.messages), 5) + + def test_many_messages(self): + """Test many messages to fill the batches with topics and non-topic + + This is a long test, intended to ensure that all the topic messages + propperly come out and set whether to include the topic flag again or + proceed to get non-topic messages. + + """ + p = self._makeFUT() + check = CheckStorageFactory(message_month=self.db.current_msg_month) + self._store_messages(check.uaid, topic=True, num=22) + self._store_messages(check.uaid, num=15) + result = p.process(check) + eq_(len(result.messages), 10) + + # Delete all the messages returned + for msg in result.messages: + notif = msg.to_WebPushNotification() + self.db.message.delete_message(notif) + + check.timestamp = result.timestamp + check.include_topic = result.include_topic + result = p.process(check) + eq_(len(result.messages), 10) + + # Delete all the messages returned + for msg in result.messages: + notif = msg.to_WebPushNotification() + self.db.message.delete_message(notif) + + check.timestamp = result.timestamp + check.include_topic = result.include_topic + result = p.process(check) + eq_(len(result.messages), 2) + + # Delete all the messages returned + for msg in result.messages: + notif = msg.to_WebPushNotification() + self.db.message.delete_message(notif) + + check.timestamp = result.timestamp + check.include_topic = result.include_topic + result = p.process(check) + eq_(len(result.messages), 10) + + check.timestamp = result.timestamp + check.include_topic = result.include_topic + result = p.process(check) + eq_(len(result.messages), 5) + + +class TestIncrementStorageProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import IncrementStorageCommand + return IncrementStorageCommand(self.conf, self.db) + + def test_inc_storage(self): + from autopush.webpush_server import CheckStorageCommand + inc_command = self._makeFUT() + check_command = CheckStorageCommand(self.conf, self.db) + check = CheckStorageFactory(message_month=self.db.current_msg_month) + uaid = check.uaid + + # First store/register some messages + self._store_messages(check.uaid, num=15) + + # Pull 10 out + check_result = check_command.process(check) + eq_(len(check_result.messages), 10) + + # We should now have an updated timestamp returned, increment it + inc = IncStoragePosition(uaid=uaid.hex, + message_month=self.db.current_msg_month, + timestamp=check_result.timestamp) + inc_command.process(inc) + + # Create a new check command, and verify we resume from 10 in + check = CheckStorageFactory( + uaid=uaid.hex, + message_month=self.db.current_msg_month + ) + check_result = check_command.process(check) + eq_(len(check_result.messages), 5) + + +class TestDeleteMessageProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import DeleteMessageCommand + return DeleteMessageCommand(self.conf, self.db) + + def test_delete_message(self): + from autopush.webpush_server import CheckStorageCommand + check_command = CheckStorageCommand(self.conf, self.db) + check = CheckStorageFactory(message_month=self.db.current_msg_month) + delete_command = self._makeFUT() + + # Store some topic messages + self._store_messages(check.uaid, topic=True, num=7) + + # Fetch them + results = check_command.process(check) + eq_(len(results.messages), 7) + + # Delete 2 of them + for notif in results.messages[:2]: + delete_command.process(DeleteMessage( + message_month=self.db.current_msg_month, + message=notif, + )) + + # Fetch messages again + results = check_command.process(check) + eq_(len(results.messages), 5) + + +class TestDropUserProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import DropUserCommand + return DropUserCommand(self.conf, self.db) + + def test_drop_user(self): + drop_command = self._makeFUT() + + # Create a user + user = UserItemFactory() + uaid = user["uaid"] + self.db.router.register_user(user) + + # Check that its there + item = self.db.router.get_uaid(uaid) + ok_(item is not None) + + # Drop it + drop_command.process(DropUser(uaid=uaid)) + + # Verify its gone + with assert_raises(ItemNotFound): + self.db.router.get_uaid(uaid) + + +class TestMigrateUserProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import MigrateUserCommand + return MigrateUserCommand(self.conf, self.db) + + def test_migrate_user(self): + migrate_command = self._makeFUT() + + # Create a user + last_month = make_rotating_tablename("message", delta=-1) + user = UserItemFactory(current_month=last_month) + uaid = user["uaid"] + self.db.router.register_user(user) + + # Store some messages so we have some channels + self._store_messages(UUID(uaid), num=3) + + # Check that its there + item = self.db.router.get_uaid(uaid) + _, channels = self.db.message_tables[last_month].all_channels(uaid) + ok_(item["current_month"] != self.db.current_msg_month) + ok_(item is not None) + eq_(len(channels), 3) + + # Migrate it + migrate_command.process( + MigrateUser(uaid=uaid, message_month=last_month) + ) + + # Check that its in the new spot + item = self.db.router.get_uaid(uaid) + _, channels = self.db.message.all_channels(uaid) + eq_(item["current_month"], self.db.current_msg_month) + ok_(item is not None) + eq_(len(channels), 3) + + +class TestRegisterProcessor(BaseSetup): + + def _makeFUT(self): + from autopush.webpush_server import RegisterCommand + return RegisterCommand(self.conf, self.db) + + def test_register(self): + cmd = self._makeFUT() + chid = str(uuid4()) + result = cmd.process(Register( + uaid=uuid4().hex, + channel_id=chid, + message_month=self.db.current_msg_month) + ) + ok_(result.endpoint) + ok_(self.metrics.increment.called) + eq_(self.metrics.increment.call_args[0][0], 'ua.command.register') + ok_(self.logs.logged( + lambda e: (e['log_format'] == "Register" and + e['channel_id'] == chid and + e['endpoint'] == result.endpoint) + )) + + def _test_invalid(self, chid, msg="use lower case, dashed format", + status=401): + cmd = self._makeFUT() + result = cmd.process(Register( + uaid=uuid4().hex, + channel_id=chid, + message_month=self.db.current_msg_month) + ) + ok_(result.error) + ok_(msg in result.error_msg) + eq_(status, result.status) + + def test_register_bad_chid(self): + self._test_invalid("oof", "Invalid UUID") + + def test_register_bad_chid_upper(self): + self._test_invalid(str(uuid4()).upper()) + + def test_register_bad_chid_nodash(self): + self._test_invalid(uuid4().hex) + + def test_register_over_provisioning(self): + self.db.message.register_channel = Mock( + side_effect=ProvisionedThroughputExceededException(None, None)) + self._test_invalid(str(uuid4()), "overloaded", 503) + + +class TestUnregisterProcessor(BaseSetup): + + def _makeFUT(self): + from autopush.webpush_server import UnregisterCommand + return UnregisterCommand(self.conf, self.db) + + def test_unregister(self): + cmd = self._makeFUT() + chid = str(uuid4()) + result = cmd.process(Unregister( + uaid=uuid4().hex, + channel_id=chid, + message_month=self.db.current_msg_month) + ) + ok_(result.success) + ok_(self.metrics.increment.called) + eq_(self.metrics.increment.call_args[0][0], 'ua.command.unregister') + ok_(self.logs.logged( + lambda e: (e['log_format'] == "Unregister" and + e['channel_id'] == chid) + )) + + def test_unregister_bad_chid(self): + cmd = self._makeFUT() + result = cmd.process(Unregister( + uaid=uuid4().hex, + channel_id="quux", + message_month=self.db.current_msg_month) + ) + ok_(result.error) + ok_("Invalid UUID" in result.error_msg) + + +class TestStoreMessagesProcessor(BaseSetup): + def _makeFUT(self): + from autopush.webpush_server import StoreMessagesUserCommand + return StoreMessagesUserCommand(self.conf, self.db) + + def test_store_messages(self): + cmd = self._makeFUT() + store_message = StoreMessageFactory() + response = cmd.process(store_message) + eq_(response.success, True) diff --git a/autopush/webpush_server.py b/autopush/webpush_server.py new file mode 100644 index 00000000..2e949820 --- /dev/null +++ b/autopush/webpush_server.py @@ -0,0 +1,621 @@ +"""WebPush Server + +""" +from threading import Thread +from uuid import UUID, uuid4 + +import attr +from attr import ( + attrs, + attrib, +) +from boto.dynamodb2.exceptions import ItemNotFound +from boto.exception import JSONResponseError +from typing import Dict, List, Optional # noqa +from twisted.logger import Logger + +from autopush.db import ( # noqa + DatabaseManager, + has_connected_this_month, + hasher, + generate_last_connect, +) +from autopush.config import AutopushConfig # noqa +from autopush.metrics import IMetrics # noqa +from autopush.types import JSONDict # noqa +from autopush.utils import WebPushNotification +from autopush.websocket import USER_RECORD_VERSION +from autopush_rs import AutopushCall, AutopushServer, AutopushQueue # noqa + +log = Logger() + +# sentinel objects +_STOP = object() + + +# Conversion functions +def uaid_from_str(input): + # type: (Optional[str]) -> Optional[UUID] + """Parse a uaid and verify the raw input matches the hex version (no + dashes)""" + try: + uuid = UUID(input) + if uuid.hex != input: + return None + return uuid + except (TypeError, ValueError): + return None + + +@attrs(slots=True) +class WebPushMessage(object): + """Serializable version of attributes needed for message delivery""" + uaid = attrib() # type: str + timestamp = attrib() # type: int + channelID = attrib() # type: str + ttl = attrib() # type: int + topic = attrib() # type: str + version = attrib() # type: str + sortkey_timestamp = attrib(default=None) # type: Optional[int] + data = attrib(default=None) # type: Optional[str] + headers = attrib(default=None) # type: Optional[JSONDict] + + @classmethod + def from_WebPushNotification(cls, notif): + # type: (WebPushNotification) -> WebPushMessage + p = notif.websocket_format() + del p["messageType"] + return cls( + uaid=notif.uaid.hex, + timestamp=int(notif.timestamp), + sortkey_timestamp=notif.sortkey_timestamp, + ttl=int(notif.ttl) if notif.ttl else notif.ttl, + topic=notif.topic, + **p + ) + + def to_WebPushNotification(self): + # type: () -> WebPushNotification + return WebPushNotification( + uaid=UUID(self.uaid), + channel_id=self.channelID, + data=self.data, + headers=self.headers, + ttl=self.ttl, + topic=self.topic, + timestamp=self.timestamp, + message_id=self.version, + update_id=self.version, + ) + + +############################################################################### +# Input messages off the incoming queue +############################################################################### +@attrs(slots=True) +class InputCommand(object): + pass + + +@attrs(slots=True) +class Hello(InputCommand): + connected_at = attrib() # type: int + uaid = attrib(default=None, convert=uaid_from_str) # type: Optional[UUID] + + +@attrs(slots=True) +class CheckStorage(InputCommand): + uaid = attrib(convert=uaid_from_str) # type: UUID + message_month = attrib() # type: str + include_topic = attrib() # type: bool + timestamp = attrib(default=None) # type: Optional[int] + + +@attrs(slots=True) +class IncStoragePosition(InputCommand): + uaid = attrib(convert=uaid_from_str) # type: UUID + message_month = attrib() # type: str + timestamp = attrib() # type: int + + +@attrs(slots=True) +class DeleteMessage(InputCommand): + message_month = attrib() # type: str + message = attrib() # type: WebPushMessage + + +@attrs(slots=True) +class DropUser(InputCommand): + uaid = attrib(convert=uaid_from_str) # type: UUID + + +@attrs(slots=True) +class MigrateUser(InputCommand): + uaid = attrib(convert=uaid_from_str) # type: UUID + message_month = attrib() # type: str + + +@attrs(slots=True) +class StoreMessages(InputCommand): + message_month = attrib() # type: str + messages = attrib( + default=attr.Factory(list) + ) # type: List[WebPushMessage] + + +############################################################################### +# Output messages serialized to the outgoing queue +############################################################################### +@attrs(slots=True) +class OutputCommand(object): + pass + + +@attrs(slots=True) +class HelloResponse(OutputCommand): + uaid = attrib() # type: Optional[str] + message_month = attrib() # type: str + reset_uaid = attrib() # type: bool + rotate_message_table = attrib(default=False) # type: bool + + +@attrs(slots=True) +class CheckStorageResponse(OutputCommand): + include_topic = attrib() # type: bool + messages = attrib( + default=attr.Factory(list) + ) # type: List[WebPushMessage] + timestamp = attrib(default=None) # type: Optional[int] + + +@attrs(slots=True) +class IncStoragePositionResponse(OutputCommand): + success = attrib(default=True) # type: bool + + +@attrs(slots=True) +class DeleteMessageResponse(OutputCommand): + success = attrib(default=True) # type: bool + + +@attrs(slots=True) +class DropUserResponse(OutputCommand): + success = attrib(default=True) # type: bool + + +@attrs(slots=True) +class MigrateUserResponse(OutputCommand): + message_month = attrib() # type: str + + +@attrs(slots=True) +class StoreMessagesResponse(OutputCommand): + success = attrib(default=True) # type: bool + + +############################################################################### +# Main push server class +############################################################################### +class WebPushServer(object): + def __init__(self, conf, db, num_threads=10): + # type: (AutopushConfig, DatabaseManager) -> WebPushServer + self.conf = conf + self.db = db + self.db.setup_tables() + self.num_threads = num_threads + self.metrics = self.db.metrics + self.incoming = AutopushQueue() + self.workers = [] # type: List[Thread] + self.command_processor = CommandProcessor(conf, self.db) + self.rust = AutopushServer(conf, self.incoming) + self.running = False + + def run(self): + self.start() + for worker in self.workers: + worker.join() + + def start(self): + # type: (int) -> None + self.running = True + for _ in range(self.num_threads): + self.workers.append( + self._create_thread_worker( + processor=self.command_processor, + input_queue=self.incoming, + ) + ) + self.rust.startService() + + def stop(self): + self.running = False + self.rust.stopService() + + def _create_thread_worker(self, processor, input_queue): + # type: (CommandProcessor, AutopushQueue) -> Thread + def _thread_worker(): + while self.running: + call = input_queue.recv() + try: + if call is None: + break + command = call.json() + result = processor.process_message(command) + call.complete(result) + except Exception as exc: + # XXX: + import traceback + traceback.print_exc() + log.error("Exception in worker queue thread") + call.complete(dict( + error=True, + error_msg=str(exc), + )) + return self.spawn(_thread_worker) + + def spawn(self, func, *args, **kwargs): + t = Thread(target=func, args=args, kwargs=kwargs) + t.start() + return t + + +class CommandProcessor(object): + def __init__(self, conf, db): + # type: (AutopushConfig, DatabaseManager) -> CommandProcessor + self.conf = conf + self.db = db + self.hello_processor = HelloCommand(conf, db) + self.check_storage_processor = CheckStorageCommand(conf, db) + self.inc_storage_processor = IncrementStorageCommand(conf, db) + self.delete_message_processor = DeleteMessageCommand(conf, db) + self.drop_user_processor = DropUserCommand(conf, db) + self.migrate_user_proocessor = MigrateUserCommand(conf, db) + self.register_process = RegisterCommand(conf, db) + self.unregister_process = UnregisterCommand(conf, db) + self.store_messages_process = StoreMessagesUserCommand(conf, db) + self.deserialize = dict( + hello=Hello, + check_storage=CheckStorage, + inc_storage_position=IncStoragePosition, + delete_message=DeleteMessage, + drop_user=DropUser, + migrate_user=MigrateUser, + register=Register, + unregister=Unregister, + store_messages=StoreMessages, + ) + self.command_dict = dict( + hello=self.hello_processor, + check_storage=self.check_storage_processor, + inc_storage_position=self.inc_storage_processor, + delete_message=self.delete_message_processor, + drop_user=self.drop_user_processor, + migrate_user=self.migrate_user_proocessor, + register=self.register_process, + unregister=self.unregister_process, + store_messages=self.store_messages_process, + ) # type: Dict[str, ProcessorCommand] + + def process_message(self, input): + # type: (JSONDict) -> JSONDict + """Process incoming message from the Rust server""" + command = input.pop("command", None) # type: str + if command not in self.command_dict: + log.critical("No command present: %s" % command) + return dict( + error=True, + error_msg="Command not found", + ) + from pprint import pformat + log.info('command: %r %r' % (pformat(command), input)) + command_obj = self.deserialize[command](**input) + response = attr.asdict(self.command_dict[command].process(command_obj)) + log.info('response: %s' % response) + return response + + +class ProcessorCommand(object): + """Parent class for processor commands""" + def __init__(self, conf, db): + # type: (AutopushConfig, DatabaseManager) -> None + self.conf = conf + self.db = db + + @property + def metrics(self): + # type: () -> IMetrics + return self.db.metrics + + def process(self, command): + raise NotImplementedError() + + +class HelloCommand(ProcessorCommand): + def process(self, hello): + # type: (Hello) -> HelloResponse + user_item = None + # XXX: + flags = dict( + message_month=self.db.current_msg_month, + reset_uaid=False + ) + if hello.uaid: + user_item, flags = self.lookup_user(hello) + + if not user_item: + user_item = self.create_user(hello) + + # Save the UAID as register_user removes it + uaid = user_item["uaid"] # type: str + success, _ = self.db.router.register_user(user_item) + if not success: + # User has already connected more recently elsewhere + return HelloResponse(uaid=None, **flags) + + return HelloResponse(uaid=uaid, **flags) + + def lookup_user(self, hello): + # type: (Hello) -> (Optional[JSONDict], JSONDict) + flags = dict( + message_month=None, + rotate_message_table=False, + reset_uaid=False, + ) + uaid = hello.uaid.hex + try: + record = self.db.router.get_uaid(uaid) + except ItemNotFound: + return None, flags + + # All records must have a router_type and connected_at, in some odd + # cases a record exists for some users that doesn't + if "router_type" not in record or "connected_at" not in record: + self.db.router.drop_user(uaid) + return None, flags + + # Current month must exist and be a valid prior month + if ("current_month" not in record) or record["current_month"] \ + not in self.db.message_tables: + self.db.router.drop_user(uaid) + return None, flags + + # Determine if message table rotation is needed + flags["message_month"] = record["current_month"] + if record["current_month"] != self.db.current_msg_month: + flags["rotate_message_table"] = True + + # Include and update last_connect if needed, otherwise exclude + if has_connected_this_month(record): + del record["last_connect"] + else: + record["last_connect"] = generate_last_connect() + + # Determine if this is missing a record version + if ("record_version" not in record or + int(record["record_version"]) < USER_RECORD_VERSION): + flags["reset_uaid"] = True + + # Update the node_id, connected_at for this node/connected_at + record["node_id"] = self.conf.router_url + record["connected_at"] = hello.connected_at + return record, flags + + def create_user(self, hello): + # type: (Hello) -> JSONDict + return dict( + uaid=uuid4().hex, + node_id=self.conf.router_url, + connected_at=hello.connected_at, + router_type="webpush", + last_connect=generate_last_connect(), + record_version=USER_RECORD_VERSION, + current_month=self.db.current_msg_month, + ) + + +class CheckStorageCommand(ProcessorCommand): + def process(self, command): + # type: (CheckStorage) -> CheckStorageResponse + + # First, determine if there's any messages to retrieve + timestamp, messages, include_topic = self._check_storage(command) + return CheckStorageResponse( + timestamp=timestamp, + messages=messages, + include_topic=include_topic, + ) + + def _check_storage(self, command): + timestamp = None + messages = [] + message = self.db.message_tables[command.message_month] + if command.include_topic: + timestamp, messages = message.fetch_messages( + uaid=command.uaid, limit=11, + ) + + # If we have topic messages, return them immediately + messages = [WebPushMessage.from_WebPushNotification(m) + for m in messages] + if messages: + return timestamp, messages, True + + # No messages, update the command to include the last timestamp + # that was ack'd + command.timestamp = timestamp + + if not messages or command.timestamp: + timestamp, messages = message.fetch_timestamp_messages( + uaid=command.uaid, + timestamp=command.timestamp, + ) + messages = [WebPushMessage.from_WebPushNotification(m) + for m in messages] + return timestamp, messages, False + + +class IncrementStorageCommand(ProcessorCommand): + def process(self, command): + # type: (IncStoragePosition) -> IncStoragePositionResponse + message = self.db.message_tables[command.message_month] + message.update_last_message_read(command.uaid, command.timestamp) + return IncStoragePositionResponse() + + +class DeleteMessageCommand(ProcessorCommand): + def process(self, command): + # type: (DeleteMessage) -> DeleteMessageResponse + notif = command.message.to_WebPushNotification() + message = self.db.message_tables[command.message_month] + message.delete_message(notif) + return DeleteMessageResponse() + + +class DropUserCommand(ProcessorCommand): + def process(self, command): + # type: (DropUser) -> DropUserResponse + self.db.router.drop_user(command.uaid.hex) + return DropUserResponse() + + +class MigrateUserCommand(ProcessorCommand): + def process(self, command): + # type: (MigrateUser) -> MigrateUserResponse + # Get the current channels for this month + message = self.db.message_tables[command.message_month] + _, channels = message.all_channels(command.uaid.hex) + + # Get the current message month + cur_month = self.db.current_msg_month + if channels: + # Save the current channels into this months message table + msg_table = self.db.message_tables[cur_month] + msg_table.save_channels(command.uaid.hex, channels) + + # Finally, update the route message month + self.db.router.update_message_month(command.uaid.hex, cur_month) + return MigrateUserResponse(message_month=cur_month) + + +class StoreMessagesUserCommand(ProcessorCommand): + def process(self, command): + # type: (StoreMessages) -> StoreMessagesResponse + message = self.db.message_tables[command.message_month] + for m in command.messages: + if "topic" not in m: + m["topic"] = None + notif = WebPushMessage(**m).to_WebPushNotification() + message.store_message(notif) + return StoreMessagesResponse() + + +def _validate_chid(chid): + # type: (str) -> Tuple[bool, Optional[str]] + """Ensure valid channel id format for register/unregister""" + try: + result = UUID(chid) + except ValueError: + return False, "Invalid UUID specified" + if chid != str(result): + return False, "Bad UUID format, use lower case, dashed format" + return True, None + + +@attrs(slots=True) +class Register(InputCommand): + channel_id = attrib() # type: str + uaid = attrib(convert=uaid_from_str) # type: Optional[UUID] + message_month = attrib() # type: str + key = attrib(default=None) # type: str + + +@attrs(slots=True) +class RegisterResponse(OutputCommand): + endpoint = attrib() # type: str + + +@attrs(slots=True) +class RegisterErrorResponse(OutputCommand): + error_msg = attrib() # type: str + error = attrib(default=True) # type: bool + status = attrib(default=401) # type: int + + +class RegisterCommand(ProcessorCommand): + + def process(self, command): + # type: (Register) -> Union[RegisterResponse, RegisterErrorResponse] + valid, msg = _validate_chid(command.channel_id) + if not valid: + return RegisterErrorResponse(error_msg=msg) + # XXX: pauseProducing + + endpoint = self.conf.make_endpoint( + command.uaid.hex, + command.channel_id, + command.key + ) + message = self.db.message_tables[command.message_month] + try: + message.register_channel(command.uaid.hex, command.channel_id) + except JSONResponseError: + return RegisterErrorResponse(error_msg="overloaded", status=503) + # XXX: resumeProducing + + self.metrics.increment('ua.command.register') + # XXX: user/raw_agent? + log.info( + "Register", + channel_id=command.channel_id, + endpoint=endpoint, + uaid_hash=hasher(command.uaid.hex), + ) + return RegisterResponse(endpoint=endpoint) + + +@attrs(slots=True) +class Unregister(InputCommand): + channel_id = attrib() # type: str + uaid = attrib(convert=uaid_from_str) # type: Optional[UUID] + message_month = attrib() # type: str + code = attrib(default=None) # type: int + + +@attrs(slots=True) +class UnregisterResponse(OutputCommand): + success = attrib(default=True) # type: bool + + +@attrs(slots=True) +class UnregisterErrorResponse(OutputCommand): + error_msg = attrib() # type: str + error = attrib(default=True) # type: bool + status = attrib(default=401) # type: int + + +class UnregisterCommand(ProcessorCommand): + + def process(self, + command # type: Unregister + ): + # type: (...) -> Union[UnregisterResponse, UnregisterErrorResponse] + valid, msg = _validate_chid(command.channel_id) + if not valid: + return UnregisterErrorResponse(error_msg=msg) + + message = self.db.message_tables[command.message_month] + # XXX: JSONResponseError not handled (no force_retry) + message.unregister_channel(command.uaid.hex, command.channel_id) + + # XXX: Clear out any existing tracked messages for this + # channel + + self.metrics.increment('ua.command.unregister') + # XXX: user/raw_agent? + log.info( + "Unregister", + channel_id=command.channel_id, + uaid_hash=hasher(command.uaid.hex), + **dict(code=command.code) if command.code else {} + ) + return UnregisterResponse() diff --git a/autopush_rs/Cargo.lock b/autopush_rs/Cargo.lock new file mode 100644 index 00000000..541a1871 --- /dev/null +++ b/autopush_rs/Cargo.lock @@ -0,0 +1,853 @@ +[root] +name = "autopush" +version = "0.1.0" +dependencies = [ + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tungstenite 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tungstenite 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "advapi32-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.53 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bytes" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "conv" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "core-foundation" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "core-foundation-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crypt32-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "custom_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "dbghelp-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dtoa" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "env_logger" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "error-chain" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "foreign-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-cpupool" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "httparse" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazycell" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "magenta" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "magenta-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "magenta-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mime" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicase 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "magenta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "magenta-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "native-tls" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "openssl 0.9.17 (registry+https://github.com/rust-lang/crates.io-index)", + "schannel 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_cpus" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl" +version = "0.9.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "foreign-types 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-sys" +version = "0.9.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.53 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "percent-encoding" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pkg-config" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "magenta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-demangle" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "schannel" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crypt32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "secur32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scoped-tls" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "secur32-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "security-framework" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "security-framework-sys" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive_internals" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha1" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "take" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tempdir" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-dns-unofficial" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-io" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tls" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-dns-unofficial 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "tungstenite 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tungstenite" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "utf-8 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicase" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "url" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf-8" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "uuid" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "vcpkg" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum advapi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e06588080cb19d0acb6739808aafa5f26bfb2ca015b2b6370028b44cf7cb8a9a" +"checksum backtrace 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "72f9b4182546f4b04ebc4ab7f84948953a118bd6021a1b6a6c909e3e94f6be76" +"checksum backtrace-sys 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "afccc5772ba333abccdf60d55200fa3406f8c59dcf54d5f7998c9107d3799c7c" +"checksum base64 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" +"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" +"checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" +"checksum byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff81738b726f5d099632ceaffe7fb65b90212e8dce59d518729e7e8634032d3d" +"checksum bytes 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d828f97b58cc5de3e40c421d0cf2132d6b2da4ee0e11b8632fa838f0f9333ad6" +"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" +"checksum conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299" +"checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" +"checksum core-foundation-sys 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "065a5d7ffdcbc8fa145d6f0746f3555025b9097a9e9cda59f7467abae670c78d" +"checksum crypt32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e34988f7e069e0b2f3bfc064295161e489b2d4e04a2e4248fb94360cdf00b4ec" +"checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" +"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" +"checksum dtoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80c8b71fd71146990a9742fc06dcbbde19161a267e0ad4e572c35162f4578c90" +"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" +"checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" +"checksum foreign-types 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e4056b9bd47f8ac5ba12be771f77a0dae796d1bbaaf5fd0b9c2d38b69b8a29d" +"checksum futures 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4b63a4792d4f8f686defe3b39b92127fea6344de5d38202b2ee5a11bbbf29d6a" +"checksum futures-cpupool 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a283c84501e92cade5ea673a2a7ca44f71f209ccdd302a3e0896f50083d2c5ff" +"checksum gcc 0.3.53 (registry+https://github.com/rust-lang/crates.io-index)" = "e8310f7e9c890398b0e80e301c4f474e9918d2b27fca8f48486ca775fa9ffc5a" +"checksum httparse 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "af2f2dd97457e8fb1ae7c5a420db346af389926e36f43768b96f101546b04a07" +"checksum hyper 0.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "641abc3e3fcf0de41165595f801376e01106bca1fd876dda937730e477ca004c" +"checksum idna 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "014b298351066f1512874135335d62a789ffe78a9974f94b43ed5621951eaf7d" +"checksum iovec 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "29d062ee61fccdf25be172e70f34c9f6efc597e1fb8f6526e8437b2046ab26be" +"checksum itoa 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eb2f404fbc66fd9aac13e998248505e7ecb2ad8e44ab6388684c5fb11c6c251c" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" +"checksum lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3b37545ab726dd833ec6420aaba8231c5b320814b9029ad585555d2a03e94fbf" +"checksum lazycell 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3b585b7a6811fb03aa10e74b278a0f00f8dd9b45dc681f148bb29fa5cb61859b" +"checksum libc 0.2.29 (registry+https://github.com/rust-lang/crates.io-index)" = "8a014d9226c2cc402676fbe9ea2e15dd5222cd1dd57f576b5b283178c944a264" +"checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b" +"checksum magenta 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf0336886480e671965f794bc9b6fce88503563013d1bfb7a502c81fe3ac527" +"checksum magenta-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40d014c7011ac470ae28e2f76a02bfea4a8480f73e701353b49ad7a8d75f4699" +"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376" +"checksum mime 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "153f98dde2b135dece079e5478ee400ae1bab13afa52d66590eacfc40e912435" +"checksum mio 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "dbd91d3bfbceb13897065e97b2ef177a09a438cb33612b2d371bf568819a9313" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum native-tls 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04b781c9134a954c84f0594b9ab3f5606abc516030388e8511887ef4c204a1e5" +"checksum net2 0.2.31 (registry+https://github.com/rust-lang/crates.io-index)" = "3a80f842784ef6c9a958b68b7516bc7e35883c614004dd94959a4dca1b716c09" +"checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0" +"checksum num_cpus 1.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aec53c34f2d0247c5ca5d32cca1478762f301740468ee9ee6dcb7a0dd7a0c584" +"checksum openssl 0.9.17 (registry+https://github.com/rust-lang/crates.io-index)" = "085aaedcc89a2fac1eb2bc19cd66f29d4ea99fec60f82a5f3a88a6be7dbd90b5" +"checksum openssl-sys 0.9.17 (registry+https://github.com/rust-lang/crates.io-index)" = "7e3a9845a4c9fdb321931868aae5549e96bb7b979bf9af7de03603d74691b5f3" +"checksum percent-encoding 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "de154f638187706bde41d9b4738748933d64e6b37bdbffc0b47a97d16a6ae356" +"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" +"checksum rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)" = "eb250fd207a4729c976794d03db689c9be1d634ab5a1c9da9492a13d8fecbcdf" +"checksum redox_syscall 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "8312fba776a49cf390b7b62f3135f9b294d8617f7a7592cfd0ac2492b658cd7b" +"checksum rustc-demangle 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "aee45432acc62f7b9a108cc054142dac51f979e69e71ddce7d6fc7adf29e817e" +"checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" +"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum schannel 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "14a5f8491ae5fc8c51aded1f5806282a0218b4d69b1b76913a0559507e559b90" +"checksum scoped-tls 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f417c22df063e9450888a7561788e9bd46d3bb3c1466435b4eccb903807f147d" +"checksum secur32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3f412dfa83308d893101dd59c10d6fda8283465976c28c287c5c855bf8d216bc" +"checksum security-framework 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "dfa44ee9c54ce5eecc9de7d5acbad112ee58755239381f687e564004ba4a2332" +"checksum security-framework-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "5421621e836278a0b139268f36eee0dc7e389b784dc3f79d8f11aabadf41bead" +"checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" +"checksum serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "f7726f29ddf9731b17ff113c461e362c381d9d69433f79de4f3dd572488823e9" +"checksum serde_derive 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cf823e706be268e73e7747b147aa31c8f633ab4ba31f115efb57e5047c3a76dd" +"checksum serde_derive_internals 0.15.1 (registry+https://github.com/rust-lang/crates.io-index)" = "37aee4e0da52d801acfbc0cc219eb1eda7142112339726e427926a6f6ee65d3a" +"checksum serde_json 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "48b04779552e92037212c3615370f6bd57a40ebba7f20e554ff9f55e41a69a7b" +"checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" +"checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" +"checksum smallvec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +"checksum take 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" +"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" +"checksum time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "d5d788d3aa77bc0ef3e9621256885555368b47bd495c13dd2e7413c89f845520" +"checksum tokio-core 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e85d419699ec4b71bfe35bbc25bb8771e52eff0471a7f75c853ad06e200b4f86" +"checksum tokio-dns-unofficial 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "35ca396a652a11b5dd689d96dfd4d7266f9883522bc0f5d99270524b40646aad" +"checksum tokio-io 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4ab83e7adb5677e42e405fa4ceff75659d93c4d7d7dd22f52fcec59ee9f02af" +"checksum tokio-proto 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +"checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +"checksum tokio-tls 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d88e411cac1c87e405e4090be004493c5d8072a370661033b1a64ea205ec2e13" +"checksum tokio-tungstenite 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "20b4fa778e8d6d6f7c53fc097f2a5c2f3460fa5e5703d28c61309f63420f1f0b" +"checksum tungstenite 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "366c5dd8da77368bad71bb1977cfe8af094d03b87765c12d1422e2f0785afb22" +"checksum unicase 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e01da42520092d0cd2d6ac3ae69eb21a22ad43ff195676b86f8c37f487d6b80" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "51ccda9ef9efa3f7ef5d91e8f9b83bbe6955f9bf86aec89d5cce2c874625920f" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" +"checksum url 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb819346883532a271eb626deb43c4a1bb4c4dd47c519bd78137c3e72a4fe27" +"checksum utf-8 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f923c601c7ac48ef1d66f7d5b5b2d9a7ba9c51333ab75a3ddf8d0309185a56" +"checksum uuid 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" +"checksum vcpkg 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9e0a7d8bed3178a8fb112199d466eeca9ed09a14ba8ad67718179b4fd5487d0b" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/autopush_rs/Cargo.toml b/autopush_rs/Cargo.toml new file mode 100644 index 00000000..fc8724fb --- /dev/null +++ b/autopush_rs/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "autopush" +version = "0.1.0" +authors = ["Alex Crichton "] + +[lib] +crate-type = ["cdylib"] + +[dependencies] +env_logger = { version = "0.4", default-features = false } +error-chain = "0.10" +futures = "0.1" +hyper = "0.11" +libc = "0.2" +log = "0.3" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +time = "0.1" +tokio-core = "0.1" +tokio-service = "0.1" +tokio-tungstenite = "0.3" +tungstenite = "0.4" +uuid = { version = "0.5", features = ["serde", "v4"] } + +[profile.release] +debug-assertions = true diff --git a/autopush_rs/__init__.py b/autopush_rs/__init__.py new file mode 100644 index 00000000..42aa9a29 --- /dev/null +++ b/autopush_rs/__init__.py @@ -0,0 +1,133 @@ +import json + +from autopush_rs._native import ffi, lib + + +def ffi_from_buffer(s): + if s is None: + return ffi.NULL + else: + return ffi.from_buffer(s) + + +def free(obj, free_fn): + if obj.ffi is None: + return + ffi.gc(obj.ffi, None) + free_fn(obj.ffi) + obj.ffi = None + + +class AutopushServer(object): + def __init__(self, conf, queue): + # type: (AutopushConfig, AutopushQueue) -> AutopushServer + cfg = ffi.new('AutopushServerOptions*') + cfg.auto_ping_interval = conf.auto_ping_interval + cfg.auto_ping_timeout = conf.auto_ping_timeout + cfg.close_handshake_timeout = conf.close_handshake_timeout + cfg.max_connections = conf.max_connections + cfg.open_handshake_timeout = 5 + cfg.port = conf.port + cfg.ssl_cert = ffi_from_buffer(conf.ssl.cert) + cfg.ssl_dh_param = ffi_from_buffer(conf.ssl.dh_param) + cfg.ssl_key = ffi_from_buffer(conf.ssl.key) + cfg.url = ffi_from_buffer(conf.ws_url) + cfg.json_logging = True + + ptr = _call(lib.autopush_server_new, cfg) + self.ffi = ffi.gc(ptr, lib.autopush_server_free) + self.queue = queue + + def startService(self): + _call(lib.autopush_server_start, + self.ffi, + self.queue.ffi) + + def stopService(self): + if self.ffi is None: + return + _call(lib.autopush_server_stop, self.ffi) + self._free_ffi() + + def _free_ffi(self): + free(self, lib.autopush_server_free) + + +class AutopushCall: + def __init__(self, ptr): + self.ffi = ffi.gc(ptr, lib.autopush_python_call_free) + + def json(self): + msg_ptr = _call(lib.autopush_python_call_input_ptr, self.ffi) + msg_len = _call(lib.autopush_python_call_input_len, self.ffi) - 1 + buf = ffi.buffer(msg_ptr, msg_len) + return json.loads(str(buf[:])) + + def complete(self, ret): + s = json.dumps(ret) + _call(lib.autopush_python_call_complete, self.ffi, s) + self._free_ffi() + + def cancel(self): + self._free_ffi() + + def _free_ffi(self): + free(self, lib.autopush_python_call_free) + + +class AutopushQueue: + def __init__(self): + ptr = _call(lib.autopush_queue_new) + self.ffi = ffi.gc(ptr, lib.autopush_queue_free) + + def recv(self): + if self.ffi is None: + return None + ret = _call(lib.autopush_queue_recv, self.ffi) + if ffi.cast('size_t', ret) == 1: + return None + else: + return AutopushCall(ret) + + +last_err = None + + +def _call(f, *args): + # We cache errors across invocations of `_call` to avoid allocating a new + # error each time we call an FFI function. Each function call, however, + # needs a unique error, so take the global `last_err`, lazily initializing + # it if necessary. + global last_err + my_err = last_err + last_err = None + if my_err is None: + my_err = ffi.new('AutopushError*') + + # The error pointer is always the last argument, so pass that in and call + # the actual FFI function. If the return value is nonzero then it was a + # successful call and we can put our error back into the global slot + # and return. + args = args + (my_err,) + ret = f(*args) + if ffi.cast('size_t', ret) != 0: + last_err = my_err + return ret + + # If an error happened then it means that the Rust side of things panicked + # which we need to handle here. Acquire the string from the error, if + # available, and re-raise as a python `RuntimeError`. + # + # Note that we're also careful here to clean up the error's internals to + # avoid memory leaks and then once we're completely done we can restore our + # local error to its global position. + errln = lib.autopush_error_msg_len(my_err); + if errln > 0: + ptr = lib.autopush_error_msg_ptr(my_err) + msg = 'rust panic: ' + ffi.buffer(ptr, errln)[:] + exn = RuntimeError(msg) + else: + exn = RuntimeError('unknown error in rust') + lib.autopush_error_cleanup(my_err) + last_err = my_err + raise exn diff --git a/autopush_rs/src/call.rs b/autopush_rs/src/call.rs new file mode 100644 index 00000000..687bc4eb --- /dev/null +++ b/autopush_rs/src/call.rs @@ -0,0 +1,397 @@ +//! Implementation of calling methods/objects in python +//! +//! The main `Server` has a channel that goes back to the main python thread, +//! and that's used to send instances of `PythonCall` from the Rust thread to +//! the Python thread. Typically you won't work with `PythonCall` directly +//! though but rather the various methods on the `Server` struct, documented +//! below. Each method will return a `MyFuture` of the result, representing the +//! decoded value from Python. +//! +//! Implementation-wise what's happening here is that each function call into +//! Python creates a `futures::sync::oneshot`. The `Sender` half of this oneshot +//! is sent to Python while the `Receiver` half stays in Rust. Arguments sent to +//! Python are serialized as JSON and arguments are received from Python as JSON +//! as well, meaning that they're deserialized in Rust from JSON as well. + +use std::cell::RefCell; +use std::ffi::CStr; + +use futures::Future; +use futures::sync::oneshot; +use libc::c_char; +use serde::de; +use serde::ser; +use serde_json; +use time::Tm; +use uuid::Uuid; + +use errors::*; +use rt::{self, UnwindGuard, AutopushError}; +use protocol; +use server::Server; + +#[repr(C)] +pub struct AutopushPythonCall { + inner: UnwindGuard, +} + +struct Inner { + input: String, + done: RefCell>>, +} + +pub struct PythonCall { + input: String, + output: Box, +} + +#[no_mangle] +pub extern "C" fn autopush_python_call_input_ptr(call: *mut AutopushPythonCall, + err: &mut AutopushError) + -> *const u8 +{ + unsafe { + (*call).inner.catch(err, |call| { + call.input.as_ptr() + }) + } +} + +#[no_mangle] +pub extern "C" fn autopush_python_call_input_len(call: *mut AutopushPythonCall, + err: &mut AutopushError) + -> usize +{ + unsafe { + (*call).inner.catch(err, |call| { + call.input.len() + }) + } +} + +#[no_mangle] +pub extern "C" fn autopush_python_call_complete(call: *mut AutopushPythonCall, + input: *const c_char, + err: &mut AutopushError) + -> i32 +{ + unsafe { + (*call).inner.catch(err, |call| { + let input = CStr::from_ptr(input).to_str().unwrap(); + call.done.borrow_mut().take().unwrap().call(input); + }) + } +} + +#[no_mangle] +pub extern "C" fn autopush_python_call_free(call: *mut AutopushPythonCall) { + rt::abort_on_panic(|| unsafe { + Box::from_raw(call); + }) +} + +impl AutopushPythonCall { + pub fn new(call: PythonCall) -> AutopushPythonCall { + AutopushPythonCall { + inner: UnwindGuard::new(Inner { + input: call.input, + done: RefCell::new(Some(call.output)), + }), + } + } + + fn _new(input: String, f: F) -> AutopushPythonCall + where F: FnOnce(&str) + Send + 'static, + { + AutopushPythonCall { + inner: UnwindGuard::new(Inner { + input: input, + done: RefCell::new(Some(Box::new(f))), + }), + } + } +} + +trait FnBox: Send { + fn call(self: Box, input: &str); +} + +impl FnBox for F { + fn call(self: Box, input: &str) { + (*self)(input) + } +} + + +#[derive(Serialize)] +#[serde(tag = "command", rename_all = "snake_case")] +enum Call { + Hello { + connected_at: i64, + uaid: Option, + }, + + Register { + uaid: String, + channel_id: String, + message_month: String, + key: Option, + }, + + Unregister { + uaid: String, + channel_id: String, + message_month: String, + code: i32, + }, + + CheckStorage { + uaid: String, + message_month: String, + include_topic: bool, + timestamp: Option, + }, + + DeleteMessage { + message: protocol::Notification, + message_month: String, + }, + + IncStoragePosition { + uaid: String, + message_month: String, + timestamp: i64, + }, + + DropUser { + uaid: String, + }, + + MigrateUser { + uaid: String, + message_month: String, + }, + + StoreMessages { + message_month: String, + messages: Vec, + } +} + +#[derive(Deserialize)] +struct PythonError { + pub error: bool, + pub error_msg: String, +} + +#[derive(Deserialize)] +pub struct HelloResponse { + pub uaid: Option, + pub message_month: String, + pub reset_uaid: bool, + pub rotate_message_table: bool, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum RegisterResponse { + Success { + endpoint: String, + }, + + Error { + error_msg: String, + error: bool, + status: u32, + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum UnRegisterResponse { + Success { + success: bool, + }, + + Error { + error_msg: String, + error: bool, + status: u32, + } +} + +#[derive(Deserialize)] +pub struct CheckStorageResponse { + pub include_topic: bool, + pub messages: Vec, + pub timestamp: Option, +} + +#[derive(Deserialize)] +pub struct DeleteMessageResponse { + pub success: bool, +} + +#[derive(Deserialize)] +pub struct IncStorageResponse { + pub success: bool, +} + +#[derive(Deserialize)] +pub struct DropUserResponse { + pub success: bool, +} + +#[derive(Deserialize)] +pub struct MigrateUserResponse { + pub message_month: String, +} + +#[derive(Deserialize)] +pub struct StoreMessagesResponse { + pub success: bool, +} + + +impl Server { + pub fn hello(&self, connected_at: &u64, uaid: Option<&Uuid>) + -> MyFuture + { + let ms = *connected_at as i64; + let (call, fut) = PythonCall::new(&Call::Hello { + connected_at: ms, + uaid: if let Some(uuid) = uaid { Some(uuid.simple().to_string()) } else { None }, + }); + self.send_to_python(call); + return fut + } + + pub fn register(&self, uaid: String, message_month: String, channel_id: String, key: Option) + -> MyFuture + { + let (call, fut) = PythonCall::new(&Call::Register { + uaid: uaid, + message_month: message_month, + channel_id: channel_id, + key: key, + }); + self.send_to_python(call); + return fut + } + + pub fn unregister(&self, uaid: String, message_month: String, channel_id: String, code: i32) + -> MyFuture + { + let (call, fut) = PythonCall::new(&Call::Unregister { + uaid: uaid, + message_month: message_month, + channel_id: channel_id, + code: code, + }); + self.send_to_python(call); + return fut + } + + pub fn check_storage(&self, uaid: String, message_month: String, include_topic: bool, timestamp: Option) + -> MyFuture + { + let (call, fut) = PythonCall::new(&Call::CheckStorage { + uaid: uaid, + message_month: message_month, + include_topic: include_topic, + timestamp: timestamp, + }); + self.send_to_python(call); + return fut + } + + pub fn increment_storage(&self, uaid: String, message_month: String, timestamp: i64) + -> MyFuture + { + let (call, fut) = PythonCall::new(&Call::IncStoragePosition { + uaid: uaid, + message_month: message_month, + timestamp: timestamp, + }); + self.send_to_python(call); + return fut + } + + pub fn delete_message(&self, message_month: String, notif: protocol::Notification) + -> MyFuture + { + let (call, fut) = PythonCall::new(&Call::DeleteMessage { + message: notif, + message_month: message_month, + }); + self.send_to_python(call); + return fut + } + + pub fn drop_user(&self, uaid: String) -> MyFuture { + let (call, fut) = PythonCall::new(&Call::DropUser { + uaid, + }); + self.send_to_python(call); + return fut + } + + pub fn migrate_user(&self, uaid: String, message_month: String) -> MyFuture { + let (call, fut) = PythonCall::new(&Call::MigrateUser { + uaid, + message_month, + }); + self.send_to_python(call); + return fut + } + + pub fn store_messages(&self, uaid: String, message_month: String, mut messages: Vec) + -> MyFuture + { + for message in messages.iter_mut() { + message.uaid = Some(uaid.clone()); + } + let (call, fut) = PythonCall::new(&Call::StoreMessages { + message_month, + messages, + }); + self.send_to_python(call); + return fut + } + + fn send_to_python(&self, call: PythonCall) { + self.tx.send(Some(call)).expect("python went away?"); + } +} + +impl PythonCall { + fn new(input: &T) -> (PythonCall, MyFuture) + where T: ser::Serialize, + U: for<'de> de::Deserialize<'de> + 'static, + { + let (tx, rx) = oneshot::channel(); + let call = PythonCall { + input: serde_json::to_string(input).unwrap(), + output: Box::new(|json: &str| { + drop(tx.send(json_or_error(json))); + }), + }; + let rx = Box::new(rx.then(|res| { + match res { + Ok(Ok(s)) => Ok(serde_json::from_str(&s)?), + Ok(Err(e)) => Err(e), + Err(_) => Err("call canceled from python".into()), + } + })); + (call, rx) + } +} + +fn json_or_error(json: &str) -> Result { + if let Ok(err) = serde_json::from_str::(json) { + if err.error { + return Err(format!("python exception: {}", err.error_msg).into()) + } + } + Ok(json.to_string()) +} diff --git a/autopush_rs/src/client.rs b/autopush_rs/src/client.rs new file mode 100644 index 00000000..82928b9a --- /dev/null +++ b/autopush_rs/src/client.rs @@ -0,0 +1,594 @@ +//! Management of connected clients to a WebPush server +//! +//! This module is a pretty heavy work in progress. The intention is that +//! this'll house all the various state machine transitions and state management +//! of connected clients. Note that it's expected there'll be a lot of connected +//! clients, so this may appears relatively heavily optimized! + +use std::rc::Rc; + +use futures::AsyncSink; +use futures::future::{Either}; +use futures::sync::mpsc; +use futures::{Stream, Sink, Future, Poll, Async}; +use tokio_core::reactor::Timeout; +use time; +use uuid::Uuid; + +use call; +use errors::*; +use protocol::{ClientAck, ClientMessage, ServerMessage, ServerNotification, Notification}; +use server::Server; + +pub struct RegisteredClient { + pub uaid: Uuid, + pub tx: mpsc::UnboundedSender, +} + +// Represents a websocket client connection that may or may not be authenticated +pub struct Client { + data: ClientData, + state: ClientState, +} + +pub struct ClientData { + webpush: Option, + srv: Rc, + ws: T, +} + +// Represent the state for a valid WebPush client that is authenticated +pub struct WebPushClient { + uaid: Uuid, + rx: mpsc::UnboundedReceiver, + flags: ClientFlags, + message_month: String, + unacked_direct_notifs: Vec, + unacked_stored_notifs: Vec, + // Highest version from stored, retained for use with increment + // when all the unacked storeds are ack'd + unacked_stored_highest: Option, +} + +impl WebPushClient { + fn unacked_messages(&self) -> bool { + self.unacked_stored_notifs.len() > 0 || self.unacked_direct_notifs.len() > 0 + } +} + +pub struct ClientFlags { + include_topic: bool, + increment_storage: bool, + check: bool, + reset_uaid: bool, + rotate_message_table: bool, +} + +impl ClientFlags { + fn new() -> ClientFlags { + ClientFlags { + include_topic: true, + increment_storage: false, + check: true, + reset_uaid: false, + rotate_message_table: false, + } + } + + pub fn none(&self) -> bool { + // Indicate if none of the flags are true. + match *self { + ClientFlags { + include_topic: false, + increment_storage: false, + check: false, + reset_uaid: false, + rotate_message_table: false, + } => true, + _ => false, + } + } +} + +pub enum ClientState { + WaitingForHello(Timeout), + WaitingForProcessHello(MyFuture), + WaitingForRegister(Uuid, MyFuture), + WaitingForUnRegister(Uuid, MyFuture), + WaitingForCheckStorage(MyFuture), + WaitingForDelete(MyFuture), + WaitingForIncrementStorage(MyFuture), + WaitingForDropUser(MyFuture), + WaitingForMigrateUser(MyFuture), + FinishSend(Option, Option>), + SendMessages(Option>), + CheckStorage, + IncrementStorage, + WaitingForAcks, + Await, + Done, + ShutdownCleanup(Option), +} + +impl Client +where + T: Stream + + Sink + + 'static, +{ + /// Spins up a new client communicating over the websocket `ws` specified. + /// + /// The `ws` specified already has ping/pong parts of the websocket + /// protocol managed elsewhere, and this struct is only expected to deal + /// with webpush-specific messages. + /// + /// The `srv` argument is the server that this client is attached to and + /// the various state behind the server. This provides transitive access to + /// various configuration options of the server as well as the ability to + /// call back into Python. + pub fn new(ws: T, srv: &Rc) -> Client { + let srv = srv.clone(); + let timeout = Timeout::new(srv.opts.open_handshake_timeout.unwrap(), &srv.handle).unwrap(); + Client { + state: ClientState::WaitingForHello(timeout), + data: ClientData { + webpush: None, + srv: srv.clone(), + ws: ws + } + } + } + + pub fn shutdown(&mut self) { + self.data.shutdown(); + } + + fn transition(&mut self) -> Poll { + let next_state = match self.state { + ClientState::FinishSend(None, None) => { + return Err("Bad state, should not have nothing to do".into()) + }, + ClientState::FinishSend(None, ref mut next_state) => { + debug!("State: FinishSend w/next_state"); + try_ready!(self.data.ws.poll_complete()); + *next_state.take().unwrap() + } + ClientState::FinishSend(ref mut msg, ref mut next_state) => { + debug!("State: FinishSend w/msg & next_state"); + let item = msg.take().unwrap(); + let ret = self.data.ws.start_send(item).chain_err(|| "unable to send")?; + match ret { + AsyncSink::Ready => { + ClientState::FinishSend(None, Some(next_state.take().unwrap())) + }, + AsyncSink::NotReady(returned) => { + *msg = Some(returned); + return Ok(Async::NotReady); + } + } + }, + ClientState::SendMessages(ref mut more_messages) => { + debug!("State: SendMessages"); + if more_messages.is_some() { + let mut messages = more_messages.take().unwrap(); + if let Some(message) = messages.pop() { + ClientState::FinishSend( + Some(ServerMessage::Notification(message)), + Some(Box::new(ClientState::SendMessages( + if messages.len() > 0 { Some(messages) } else { None } + ))) + ) + } else { + ClientState::SendMessages( + if messages.len() > 0 { Some(messages) } else { None } + ) + } + } else { + ClientState::WaitingForAcks + } + }, + ClientState::CheckStorage => { + debug!("State: CheckStorage"); + let webpush = self.data.webpush.as_ref().unwrap(); + ClientState::WaitingForCheckStorage(self.data.srv.check_storage( + webpush.uaid.simple().to_string(), + webpush.message_month.clone(), + webpush.flags.include_topic, + webpush.unacked_stored_highest, + )) + }, + ClientState::IncrementStorage => { + debug!("State: IncrementStorage"); + let webpush = self.data.webpush.as_ref().unwrap(); + ClientState::WaitingForIncrementStorage(self.data.srv.increment_storage( + webpush.uaid.simple().to_string(), + webpush.message_month.clone(), + webpush.unacked_stored_highest.unwrap(), + )) + }, + ClientState::WaitingForHello(ref mut timeout) => { + debug!("State: WaitingForHello"); + let uaid = match try_ready!(self.data.input_with_timeout(timeout)) { + ClientMessage::Hello { uaid, use_webpush: Some(true), ..} => uaid, + _ => return Err("Invalid message, must be hello".into()), + }; + let ms_time = time::precise_time_ns() / 1000; + ClientState::WaitingForProcessHello(self.data.srv.hello(&ms_time, uaid.as_ref())) + }, + ClientState::WaitingForProcessHello(ref mut response) => { + debug!("State: WaitingForProcessHello"); + match try_ready!(response.poll()) { + call::HelloResponse { uaid: Some(uaid), message_month, reset_uaid, rotate_message_table } + => self.data.process_hello(uaid, message_month, reset_uaid, rotate_message_table), + _ => return Err("Already connected elsewhere".into()), + } + }, + ClientState::WaitingForCheckStorage(ref mut response) => { + debug!("State: WaitingForCheckStorage"); + let (include_topic, mut messages, timestamp) = match try_ready!(response.poll()) { + call::CheckStorageResponse { include_topic, messages, timestamp } + => (include_topic, messages, timestamp), + }; + debug!("Got checkstorage response"); + let webpush = self.data.webpush.as_mut().unwrap(); + webpush.flags.include_topic = include_topic; + webpush.unacked_stored_highest = timestamp; + if messages.len() > 0 { + webpush.flags.increment_storage = !include_topic; + webpush.unacked_stored_notifs.extend(messages.iter().cloned()); + let message = ServerMessage::Notification( + messages.pop().unwrap() + ); + ClientState::FinishSend( + Some(message), + Some(Box::new(ClientState::SendMessages(Some(messages)))) + ) + } else { + webpush.flags.check = false; + ClientState::Await + } + }, + ClientState::WaitingForIncrementStorage(ref mut response) => { + debug!("State: WaitingForIncrementStorage"); + try_ready!(response.poll()); + self.data.webpush.as_mut().unwrap().flags.increment_storage = false; + ClientState::WaitingForAcks + }, + ClientState::WaitingForMigrateUser(ref mut response) => { + debug!("State: WaitingForMigrateUser"); + let message_month = match try_ready!(response.poll()) { + call::MigrateUserResponse{ message_month} => message_month + }; + let webpush = self.data.webpush.as_mut().unwrap(); + webpush.message_month = message_month; + webpush.flags.rotate_message_table = false; + ClientState::Await + }, + ClientState::WaitingForRegister(channel_id, ref mut response) => { + debug!("State: WaitingForRegister"); + let msg = match try_ready!(response.poll()) { + call::RegisterResponse::Success { endpoint } => { + ServerMessage::Register { + channel_id: channel_id, + status: 200, + push_endpoint: endpoint, + } + }, + call::RegisterResponse::Error { error_msg, status, .. } => { + debug!("Got unregister fail, error: {}", error_msg); + ServerMessage::Register { + channel_id: channel_id, + status: status, + push_endpoint: "".into(), + } + } + }; + let next_state = if self.data.unacked_messages() { + ClientState::WaitingForAcks + } else { + ClientState::Await + }; + ClientState::FinishSend(Some(msg), Some(Box::new(next_state))) + }, + ClientState::WaitingForUnRegister(channel_id, ref mut response) => { + debug!("State: WaitingForUnRegister"); + let msg = match try_ready!(response.poll()) { + call::UnRegisterResponse::Success{ success } => { + debug!("Got the unregister response"); + ServerMessage::Unregister { + channel_id: channel_id, + status: if success { 200 } else { 500 }, + } + }, + call::UnRegisterResponse::Error { error_msg, status, .. } => { + debug!("Got unregister fail, error: {}", error_msg); + ServerMessage::Unregister { channel_id, status } + } + }; + let next_state = if self.data.unacked_messages() { + ClientState::WaitingForAcks + } else { + ClientState::Await + }; + ClientState::FinishSend(Some(msg), Some(Box::new(next_state))) + }, + ClientState::WaitingForAcks => { + debug!("State: WaitingForAcks"); + if let Some(next_state) = self.data.determine_acked_state() { + return Ok(next_state.into()) + } + match try_ready!(self.data.input()) { + ClientMessage::Register { channel_id, key } => { + self.data.process_register(channel_id, key) + }, + ClientMessage::Unregister { channel_id, code } => { + self.data.process_unregister(channel_id, code) + }, + ClientMessage::Ack { updates } => { + self.data.process_acks(updates) + } + _ => return Err("Invalid state transition".into()) + } + }, + ClientState::WaitingForDelete(ref mut response) => { + debug!("State: WaitingForDelete"); + try_ready!(response.poll()); + ClientState::WaitingForAcks + }, + ClientState::WaitingForDropUser(ref mut response) => { + debug!("State: WaitingForDropUser"); + try_ready!(response.poll()); + ClientState::Done + }, + ClientState::Await => { + debug!("State: Await"); + if self.data.webpush.as_ref().unwrap().flags.check { + return Ok(ClientState::CheckStorage.into()) + } + match try_ready!(self.data.input_or_notif()) { + Either::A(ClientMessage::Register { channel_id, key }) => { + self.data.process_register(channel_id, key) + }, + Either::A(ClientMessage::Unregister { channel_id, code }) => { + self.data.process_unregister(channel_id, code) + }, + Either::B(ServerNotification::Notification(notif)) => { + let webpush = self.data.webpush.as_mut().unwrap(); + webpush.unacked_direct_notifs.push(notif.clone()); + debug!("Got a notification to send, sending!"); + ClientState::FinishSend( + Some(ServerMessage::Notification(notif)), + Some(Box::new(ClientState::WaitingForAcks)), + ) + }, + Either::B(ServerNotification::CheckStorage) => { + let webpush = self.data.webpush.as_mut().unwrap(); + webpush.flags.include_topic = true; + webpush.flags.check = true; + ClientState::Await + }, + _ => return Err("Invalid message".into()) + } + }, + ClientState::ShutdownCleanup(ref mut err) => { + debug!("State: ShutdownCleanup"); + if let Some(err_obj) = err.take() { + debug!("Error for shutdown: {}", err_obj); + }; + self.data.shutdown(); + ClientState::Done + }, + _ => return Err("Transition didn't work".into()), + }; + Ok(next_state.into()) + } +} + +impl ClientData +where + T: Stream + + Sink + + 'static, +{ + fn input(&mut self) -> Poll { + let item = match self.ws.poll()? { + Async::Ready(None) => return Err("Client dropped".into()), + Async::Ready(Some(msg)) => Async::Ready(msg), + Async::NotReady => Async::NotReady + }; + Ok(item) + } + + fn input_with_timeout(&mut self, timeout: &mut Timeout) -> Poll { + let item = match timeout.poll()? { + Async::Ready(_) => return Err("Client timed out".into()), + Async::NotReady => { + match self.ws.poll()? { + Async::Ready(None) => return Err("Client dropped".into()), + Async::Ready(Some(msg)) => Async::Ready(msg), + Async::NotReady => Async::NotReady, + } + } + }; + Ok(item) + } + + fn input_or_notif(&mut self) -> Poll, Error> { + let webpush = self.webpush.as_mut().unwrap(); + let item = match webpush.rx.poll() { + Ok(Async::Ready(Some(notif))) => Either::B(notif), + Ok(Async::Ready(None)) => return Err("Sending side went byebye".into()), + Ok(Async::NotReady) => { + match self.ws.poll()? { + Async::Ready(None) => return Err("Client dropped".into()), + Async::Ready(Some(msg)) => Either::A(msg), + Async::NotReady => return Ok(Async::NotReady), + } + }, + Err(_) => return Err("Unexpected error".into()), + }; + Ok(Async::Ready(item)) + } + + fn process_hello(&mut self, uaid: Uuid, message_month: String, reset_uaid: bool, rotate_message_table: bool) -> ClientState { + let (tx, rx) = mpsc::unbounded(); + let mut flags = ClientFlags::new(); + flags.reset_uaid = reset_uaid; + flags.rotate_message_table = rotate_message_table; + self.webpush = Some(WebPushClient { + uaid: uaid, + flags: flags, + rx: rx, + message_month: message_month, + unacked_direct_notifs: Vec::new(), + unacked_stored_notifs: Vec::new(), + unacked_stored_highest: None, + }); + self.srv.connect_client(RegisteredClient { uaid: uaid, tx: tx }); + let response = ServerMessage::Hello { + uaid: uaid.hyphenated().to_string(), + status: 200, + use_webpush: Some(true), + }; + ClientState::FinishSend(Some(response), Some(Box::new(ClientState::Await))) + } + + fn process_register(&mut self, channel_id: Uuid, key: Option) -> ClientState { + debug!("Got a register command"); + let webpush = self.webpush.as_ref().unwrap(); + let uaid = webpush.uaid.clone(); + let message_month = webpush.message_month.clone(); + let channel_id_str = channel_id.hyphenated().to_string(); + let fut = self.srv.register( + uaid.simple().to_string(), + message_month, + channel_id_str, + key, + ); + ClientState::WaitingForRegister(channel_id, fut) + } + + fn process_unregister(&mut self, channel_id: Uuid, code: Option) -> ClientState { + debug!("Got a unregister command"); + let webpush = self.webpush.as_ref().unwrap(); + let uaid = webpush.uaid.clone(); + let message_month = webpush.message_month.clone(); + let channel_id_str = channel_id.hyphenated().to_string(); + let fut = self.srv.unregister( + uaid.simple().to_string(), + message_month, + channel_id_str, + code.unwrap_or(200), + ); + ClientState::WaitingForUnRegister(channel_id, fut) + } + + fn process_acks(&mut self, updates: Vec) -> ClientState { + let webpush = self.webpush.as_mut().unwrap(); + let mut fut: Option> = None; + for notif in updates.iter() { + if let Some(pos) = webpush.unacked_direct_notifs.iter().position(|v| { + v.channel_id == notif.channel_id && v.version == notif.version + }) { + webpush.unacked_direct_notifs.remove(pos); + continue; + }; + if let Some(pos) = webpush.unacked_stored_notifs.iter().position(|v| { + v.channel_id == notif.channel_id && v.version == notif.version + }) { + let message_month = webpush.message_month.clone(); + let n = webpush.unacked_stored_notifs.remove(pos); + if n.topic.is_some() { + if fut.is_none() { + fut = Some(self.srv.delete_message(message_month, n)) + } else { + let my_fut = self.srv.delete_message(message_month, n); + fut = Some(Box::new(fut.take().unwrap().and_then(move |_| { + my_fut + }))); + } + } + continue; + }; + } + if let Some(my_fut) = fut { + ClientState::WaitingForDelete(my_fut) + } else { + ClientState::WaitingForAcks + } + } + + // Called from WaitingForAcks to determine if we're in fact done waiting for acks + // and to determine where we might go next + fn determine_acked_state(&mut self) -> Option { + let webpush = self.webpush.as_ref().unwrap(); + let all_acked = !self.unacked_messages(); + if all_acked && webpush.flags.check && webpush.flags.increment_storage { + Some(ClientState::IncrementStorage) + } else if all_acked && webpush.flags.check { + Some(ClientState::CheckStorage) + } else if all_acked && webpush.flags.rotate_message_table { + Some(ClientState::WaitingForMigrateUser( + self.srv.migrate_user( + webpush.uaid.simple().to_string(), + webpush.message_month.clone(), + ) + )) + } else if all_acked && webpush.flags.reset_uaid { + Some(ClientState::WaitingForDropUser( + self.srv.drop_user(webpush.uaid.simple().to_string()) + )) + } else if all_acked && webpush.flags.none() { + Some(ClientState::Await) + } else { + None + } + } + + fn unacked_messages(&self) -> bool { + self.webpush.as_ref().unwrap().unacked_messages() + } + + pub fn shutdown(&mut self) { + // If we made it past hello, do more cleanup + if self.webpush.is_some() { + let webpush = self.webpush.take().unwrap(); + // If there's direct unack'd messages, they need to be saved out without blocking + // here + self.srv.disconnet_client(&webpush.uaid); + if webpush.unacked_direct_notifs.len() > 0 { + self.srv.handle.spawn(self.srv.store_messages( + webpush.uaid.simple().to_string(), + webpush.message_month, + webpush.unacked_direct_notifs + ).then(|_| { + debug!("Finished saving unacked direct notifications"); + Ok(()) + })) + } + }; + } +} + +impl Future for Client +where + T: Stream + + Sink + + 'static, +{ + type Item = (); + type Error = Error; + + fn poll(&mut self) -> Poll<(), Error> { + loop { + if let ClientState::Done = self.state { + return Ok(().into()); + } + match self.transition() { + Ok(Async::NotReady) => return Ok(Async::NotReady), + Ok(Async::Ready(next_state)) => self.state = next_state, + Err(e) => self.state = ClientState::ShutdownCleanup(Some(e)), + }; + } + } +} diff --git a/autopush_rs/src/errors.rs b/autopush_rs/src/errors.rs new file mode 100644 index 00000000..abdddd1e --- /dev/null +++ b/autopush_rs/src/errors.rs @@ -0,0 +1,66 @@ +//! Error handling for Rust +//! +//! This module defines various utilities for handling errors in the Rust +//! thread. This uses the `error-chain` crate to ergonomically define errors, +//! enable them for usage with `?`, and otherwise give us some nice utilities. +//! It's expected that this module is always glob imported: +//! +//! use errors::*; +//! +//! And functions in general should then return `Result<()>`. You can add extra +//! error context via `chain_err`: +//! +//! let e = some_function_returning_a_result().chain_err(|| { +//! "some extra context here to make a nicer error" +//! })?; +//! +//! And you can also use the `MyFuture` type alias for "nice" uses of futures +//! +//! fn add(a: i32) -> MyFuture { +//! // .. +//! } +//! +//! You can find some more documentation about this in the `error-chain` crate +//! online. + +use std::any::Any; +use std::error; +use std::io; + +use tungstenite; +use serde_json; +use futures::Future; + +error_chain! { + foreign_links { + Ws(tungstenite::Error); + Io(io::Error); + Json(serde_json::Error); + } + + errors { + Thread(payload: Box) { + description("thread panicked") + } + } +} + +pub type MyFuture = Box>; + +pub trait FutureChainErr { + fn chain_err(self, callback: F) -> MyFuture + where F: FnOnce() -> E + 'static, + E: Into; +} + +impl FutureChainErr for F + where F: Future + 'static, + F::Error: error::Error + Send + 'static, +{ + fn chain_err(self, callback: C) -> MyFuture + where C: FnOnce() -> E + 'static, + E: Into, + { + Box::new(self.then(|r| r.chain_err(callback))) + } +} diff --git a/autopush_rs/src/http.rs b/autopush_rs/src/http.rs new file mode 100644 index 00000000..40877a60 --- /dev/null +++ b/autopush_rs/src/http.rs @@ -0,0 +1,67 @@ +//! Dummy module that's very likely to get entirely removed +//! +//! For now just a small HTTP server used to send notifications to our dummy +//! clients. + +use std::str; +use std::rc::Rc; + +use futures::future::err; +use futures::{Stream, Future}; +use hyper::Method; +use hyper; +use serde_json; +use tokio_service::Service; +use uuid::Uuid; + +use server::Server; + +pub struct Push(pub Rc); + +impl Service for Push { + type Request = hyper::Request; + type Response = hyper::Response; + type Error = hyper::Error; + type Future = Box>; + + fn call(&self, req: hyper::Request) -> Self::Future { + if *req.method() != Method::Put && *req.method() != Method::Post { + println!("not a PUT: {}", req.method()); + return Box::new(err(hyper::Error::Method)) + } + if req.uri().path().len() == 0 { + println!("empty uri path"); + return Box::new(err(hyper::Error::Incomplete)) + } + let req_uaid = req.uri().path()[6..].to_string(); + let uaid = match Uuid::parse_str(&req_uaid) { + Ok(id) => id, + Err(_) => { + println!("uri not uuid: {}", req_uaid); + return Box::new(err(hyper::Error::Status)) + } + }; + + debug!("Got a message, now to do something!"); + + let body = req.body().concat2(); + let srv = self.0.clone(); + Box::new(body.and_then(move |body| { + let s = String::from_utf8(body.to_vec()).unwrap(); + if let Ok(msg) = serde_json::from_str(&s) { + match srv.notify_client(uaid, msg) { + Ok(_) => return Ok(hyper::Response::new() + .with_status(hyper::StatusCode::Ok) + ), + _ => return Ok(hyper::Response::new() + .with_status(hyper::StatusCode::BadRequest) + .with_body("Unable to decode body payload") + ) + } + } + Ok(hyper::Response::new() + .with_status(hyper::StatusCode::NotFound) + ) + })) + } +} diff --git a/autopush_rs/src/lib.rs b/autopush_rs/src/lib.rs new file mode 100644 index 00000000..a756e874 --- /dev/null +++ b/autopush_rs/src/lib.rs @@ -0,0 +1,94 @@ +//! WIP: Implementation of WebPush ("autopush" as well) in Rust +//! +//! This crate currently provides an implementation of an asynchronous WebPush +//! server which is intended to be interfaced with from Python. The crate mostly +//! has a C API which is driven from `__init__.py` in Python and orchestrated +//! from Python. This is currently done to help ease the transition from the old +//! Python implementation to the new Rust implementation. Currently there's a +//! good bit of API calls to remote services still implemented in Python, but +//! the thinking is that over time these services will be rewritten in to Rust +//! and the Python codebase will shrink. +//! +//! In any case though, this'll focus mainly on the Rust bits rather than the +//! Python bits! It's worth nothing though that this crate is intended to be +//! used with `cffi` in Python, which is "seamlessly" worked with through the +//! `snaek` Python dependency. That basically just means that Python "headers" +//! for this Rust crate are generated automatically. +//! +//! # High level overview +//! +//! At 10,000 feet the general architecture here is that the main Python thread +//! spins up a Rust thread which actually does all the relevant I/O. The one +//! Rust thread uses a `Core` from `tokio-core` to perform all I/O and schedule +//! asynchronous tasks. The `tungstenite` crate is used to parse and manage the +//! WebSocket protocol, with `tokio_tungstenite` being a nicer wrapper for +//! futures-style APIs. +//! +//! The entire server is written in an asynchronous fashion using the `futures` +//! crate in Rust. This basically just means that everything is exposed as a +//! future (similar to the concept in other languages) and that's how bits and +//! pieces are chained together. +//! +//! Each connected client maintains a state machine of where it is in the +//! webpush protocol (see `states.dot` at the root of this repository). Note +//! that not all states are implemented yet, this is a work in progress! All I/O +//! is managed by Rust and various state transitions are managed by Rust as +//! well. Movement between states happens typically as a result of calls into +//! Python. The various operations here will call into Python to do things like +//! db/HTTP requests and then the results are interpreted in Rust to progress +//! the state machine. +//! +//! # Module index +//! +//! There's a number of modules that currently make up the Rust implementation, +//! and one-line summaries of these are: +//! +//! * `queue` - a MPMC queue which is used to send messages to Python and Python +//! uses to delegate work to worker threads. +//! * `server` - the main bindings for the WebPush server, where the tokio +//! `Core` is created and managed inside of the Rust thread. +//! * `client` - this is where the state machine for each connected client is +//! located, managing connections over time and sending out notifications as +//! they arrive. +//! * `protocol` - a definition of the WebPush protocol messages which are send +//! over websockets. +//! * `call` - definitions of various calls that can be made into Python, each +//! of which returning a future of the response. +//! +//! Other modules tend to be miscellaneous implementation details and likely +//! aren't as relevant to the WebPush implementation. +//! +//! Otherwise be sure to check out each module for more documentation! + +extern crate env_logger; +#[macro_use] +extern crate futures; +extern crate hyper; +extern crate libc; +#[macro_use] +extern crate log; +extern crate serde; +#[macro_use] +extern crate serde_derive; +extern crate serde_json; +extern crate time; +extern crate tokio_core; +extern crate tokio_service; +extern crate tokio_tungstenite; +extern crate tungstenite; +extern crate uuid; + +#[macro_use] +extern crate error_chain; + +mod client; +mod errors; +mod http; +mod protocol; +mod util; + +#[macro_use] +pub mod rt; +pub mod call; +pub mod server; +pub mod queue; diff --git a/autopush_rs/src/protocol.rs b/autopush_rs/src/protocol.rs new file mode 100644 index 00000000..c705658e --- /dev/null +++ b/autopush_rs/src/protocol.rs @@ -0,0 +1,95 @@ +//! Definition of Internal Router, Python, and Websocket protocol messages +//! +//! This module is a structured definition of several protocol. Both +//! messages received from the client and messages sent from the server are +//! defined here. The `derive(Deserialize)` and `derive(Serialize)` annotations +//! are used to generate the ability to serialize these structures to JSON, +//! using the `serde` crate. More docs for serde can be found at +//! https://serde.rs + +use std::collections::HashMap; +use uuid::Uuid; + +// Used for the server to flag a webpush client to deliver a Notification or Check storage +pub enum ServerNotification { + CheckStorage, + Notification(Notification), +} + +#[derive(Deserialize)] +#[serde(tag = "messageType", rename_all = "lowercase")] +pub enum ClientMessage { + Hello { + uaid: Option, + #[serde(rename = "channelIDs", skip_serializing_if = "Option::is_none")] + channel_ids: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + use_webpush: Option, + }, + + Register { + #[serde(rename = "channelID")] + channel_id: Uuid, + key: Option, + }, + + Unregister { + #[serde(rename = "channelID")] + channel_id: Uuid, + code: Option, + }, + + Ack { + updates: Vec, + }, +} + +#[derive(Deserialize)] +pub struct ClientAck { + #[serde(rename = "channelID")] + pub channel_id: Uuid, + pub version: String, +} + +#[derive(Serialize)] +#[serde(tag = "messageType", rename_all = "lowercase")] +pub enum ServerMessage { + Hello { + uaid: String, + status: u32, + #[serde(skip_serializing_if = "Option::is_none")] + use_webpush: Option, + }, + + Register { + #[serde(rename = "channelID")] + channel_id: Uuid, + status: u32, + #[serde(rename = "pushEndpoint")] + push_endpoint: String, + }, + + Unregister { + #[serde(rename = "channelID")] + channel_id: Uuid, + status: u32, + }, + + Notification(Notification), +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct Notification { + pub uaid: Option, + #[serde(rename = "channelID")] + pub channel_id: Uuid, + pub version: String, + pub ttl: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub topic: Option, + pub timestamp: u64, + #[serde(skip_serializing_if = "Option::is_none")] + data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + headers: Option> +} diff --git a/autopush_rs/src/queue.rs b/autopush_rs/src/queue.rs new file mode 100644 index 00000000..28acc48a --- /dev/null +++ b/autopush_rs/src/queue.rs @@ -0,0 +1,80 @@ +//! Thread-safe MPMC queue for working with Python and Rust +//! +//! This is created in Python and shared amongst a number of worker threads for +//! the receiving side, and then the sending side is done by the Rust thread +//! pushing requests over to Python. A `Sender` here is saved off in the +//! `Server` for sending messages. + +use std::sync::mpsc; +use std::sync::Mutex; + +use call::{AutopushPythonCall, PythonCall}; +use rt::{self, AutopushError}; + +#[repr(C)] +pub struct AutopushQueue { + tx: Mutex, + rx: Mutex>>>, +} + +pub type Sender = mpsc::Sender>; + +fn _assert_kinds() { + fn _assert() {} + _assert::(); +} + +#[no_mangle] +pub extern "C" fn autopush_queue_new(err: &mut AutopushError) + -> *mut AutopushQueue +{ + rt::catch(err, || { + let (tx, rx) = mpsc::channel(); + + Box::new(AutopushQueue { + tx: Mutex::new(tx), + rx: Mutex::new(Some(rx)), + }) + }) +} + +#[no_mangle] +pub extern "C" fn autopush_queue_recv(queue: *mut AutopushQueue, + err: &mut AutopushError) + -> *mut AutopushPythonCall +{ + rt::catch(err, || unsafe { + let mut rx = (*queue).rx.lock().unwrap(); + let msg = match *rx { + // this can't panic because we hold a reference to at least one + // sender, so it'll always block waiting for the next message + Some(ref rx) => rx.recv().unwrap(), + + // the "done" message was received by someone else, so we just keep + // propagating that + None => return None, + }; + match msg { + Some(msg) => Some(Box::new(AutopushPythonCall::new(msg))), + + // the senders are done, so all future calls shoudl bail out + None => { + *rx = None; + None + } + } + }) +} + +#[no_mangle] +pub extern "C" fn autopush_queue_free(queue: *mut AutopushQueue) { + rt::abort_on_panic(|| unsafe { + Box::from_raw(queue); + }) +} + +impl AutopushQueue { + pub fn tx(&self) -> Sender { + self.tx.lock().unwrap().clone() + } +} diff --git a/autopush_rs/src/rt.rs b/autopush_rs/src/rt.rs new file mode 100644 index 00000000..f982df27 --- /dev/null +++ b/autopush_rs/src/rt.rs @@ -0,0 +1,302 @@ +//! Runtime support for calling in and out of Python +//! +//! This module provides a number of utilities for interfacing with Python in a +//! safe fashion. It's primarily used to handle *panics* in Rust which otherwise +//! could cause segfaults or strange crashes if otherwise unhandled. +//! +//! The current protocol for Python calling into Rust looks like so: +//! +//! * Primarily, all panics are caught in Rust. Panics are intended to be +//! translated to exceptions in Python to indicate a fatal error happened in +//! Rust. +//! +//! * Almost all FFI functions take a `&mut AutopushError` as their last +//! argument. This argument is used to capture the reason of a panic so it can +//! later be introspected in Python to generate a runtime assertion. The +//! handling of `AutopushError` is intended to be relatively transparent by +//! just needing to pass it to some functions in this module. +//! +//! * A `UnwindGuard` is provided for stateful objects persisted across FFI +//! function calls. If a Rust function panics it's typically not intended to +//! be rerun at a later date with the same arguments, so what `UnwindGuard` +//! will do is only provide access to the internals *until* a panic happens. +//! After a panic then access to the internals will be gated and forbidden +//! until destruction. This should help prevent bugs from becoming worse bugs +//! quickly (in theory). +//! +//! All Rust objects shared with Python have an `UnwindGuard` internally which +//! protects all of the state that Rust is fiddling with. +//! +//! Typically you can just look at some other examples of `#[no_mangle]` +//! functions throughout this crate and copy those idioms, otherwise there's +//! documentation on each specific function here. + +use std::panic; +use std::ptr; +use std::mem; +use std::any::Any; +use std::cell::Cell; + +/// Generic error which is used on all function calls from Python into Rust. +/// +/// This is allocated in Python and reused across function calls when possible. +/// It effectively stores a `Box` which is what's created whenever a Rust +/// thread panics. This `Box` may store an object, a string, etc. +#[repr(C)] +pub struct AutopushError { + p1: usize, + p2: usize, +} + +impl AutopushError { + /// Attempts to extract the error message out of this inernal `Box`. + /// This may fail if the `Any` doesn't look like it can be stringified + /// though. + fn string(&self) -> Option<&str> { + assert!(self.p1 != 0); + assert!(self.p2 != 0); + let any: &Any = unsafe { + mem::transmute((self.p1, self.p2)) + }; + // Similar to what libstd does, only check for `&'static str` and + // `String`. + any.downcast_ref::<&'static str>() + .map(|s| &s[..]) + .or_else(|| { + any.downcast_ref::().map(|s| &s[..]) + }) + } + + fn assert_empty(&self) { + assert_eq!(self.p1, 0); + assert_eq!(self.p2, 0); + } + + fn fill(&mut self, any: Box) { + self.assert_empty(); + unsafe { + let ptrs: (usize, usize) = mem::transmute(any); + self.p1 = ptrs.0; + self.p2 = ptrs.1; + } + } + + /// Deallocates the internal `Box`, freeing the resources behind it. + unsafe fn cleanup(&mut self) { + mem::transmute::<_, Box>((self.p1, self.p2)); + self.p1 = 0; + self.p2 = 0; + } +} + +/// Acquires the length of the error message in this error, or returns 0 if +/// there is no error message. +#[no_mangle] +pub extern "C" fn autopush_error_msg_len(err: *const AutopushError) -> usize { + abort_on_panic(|| unsafe { + (*err).string().map(|s| s.len()).unwrap_or(0) + }) +} + +/// Returns the data pointer of the error message, if any. If not present +/// returns null. +#[no_mangle] +pub extern "C" fn autopush_error_msg_ptr(err: *const AutopushError) -> *const u8 { + abort_on_panic(|| unsafe { + (*err).string().map(|s| s.as_ptr()).unwrap_or(ptr::null()) + }) +} + +/// Deallocates the internal `Box`, freeing any resources it contains. +/// +/// The error itself can continue to be reused for future function calls. +#[no_mangle] +pub unsafe extern "C" fn autopush_error_cleanup(err: *mut AutopushError) { + abort_on_panic(|| { + (&mut *err).cleanup(); + }); +} + +/// Helper structure to provide "unwind safety" to ensure we don't reuse values +/// accidentally after a panic. +pub struct UnwindGuard { + poisoned: Cell, + inner: T, +} + +impl UnwindGuard { + pub fn new(t: T) -> UnwindGuard { + UnwindGuard { + poisoned: Cell::new(false), + inner: t, + } + } + + /// This function is intended to be immediately called in an FFI callback, + /// and will execute the closure `f` catching panics. + /// + /// The `err` provided will be filled in if the function panics. + /// + /// The closure `f` will execute with the state this `UnwindGuard` is + /// internally protecting, allowing it shared access to the various pieces. + /// The closure's return value is then also automatically converted to an + /// FFI-safe value through the `AbiInto` trait. Various impls for this trait + /// can be found below (possible types to return). + /// + /// Note that if this `UnwindGuard` previously caught a panic then the + /// closure `f` will not be executed. This function will immediately return + /// with the "null" return value to propagate the panic again. + pub fn catch(&self, err: &mut AutopushError, f: F) -> R::AbiRet + where F: FnOnce(&T) -> R, + R: AbiInto, + { + err.assert_empty(); + if self.poisoned.get() { + err.fill(Box::new(String::from("accessing poisoned object"))); + return R::null() + } + + // The usage of `AssertUnwindSafe` should be ok here because as + // soon as we see this closure panic we'll disallow all further + // access to the internals of `self`. + let mut panicked = true; + let ret = catch(err, panic::AssertUnwindSafe(|| { + let ret = f(&self.inner); + panicked = false; + return ret + })); + if panicked { + self.poisoned.set(true); + } + return ret + } +} + +/// Catches a panic within the closure `f`, filling in `err` if a panic happens. +/// +/// This is typically only used for constructors where there's no state +/// persisted across calls. +pub fn catch(err: &mut AutopushError, f: F) -> T::AbiRet + where F: panic::UnwindSafe + FnOnce() -> T, + T: AbiInto, +{ + err.assert_empty(); + + match panic::catch_unwind(f) { + Ok(t) => t.abi_into(), + Err(e) => unsafe { + let ptrs: (usize, usize) = mem::transmute(e); + err.p1 = ptrs.0; + err.p2 = ptrs.1; + T::null() + } + } +} + +/// Helper to *abort* on panics rather than catch them and communicate to +/// python. +/// +/// This should be rarely used but is used when executing destructors in Rust, +/// which should be infallible (and this is just a double-check that they are). +pub fn abort_on_panic(f: F) -> R + where F: FnOnce() -> R, +{ + struct Bomb { + active: bool, + } + + impl Drop for Bomb { + fn drop(&mut self) { + if self.active { + panic!("unexpected panic, aborting process"); + } + } + } + + let mut bomb = Bomb { active: true }; + let r = f(); + bomb.active = false; + return r +} + +pub trait AbiInto { + type AbiRet; + + fn abi_into(self) -> Self::AbiRet; + fn null() -> Self::AbiRet; +} + +impl AbiInto for () { + type AbiRet = i32; + + fn abi_into(self) -> i32 { + 1 + } + + fn null() -> i32 { + 0 + } +} + +impl AbiInto for Box { + type AbiRet = *mut T; + + fn abi_into(self) -> *mut T { + Box::into_raw(self) + } + + fn null() -> *mut T { + ptr::null_mut() + } +} + +impl AbiInto for Option> { + type AbiRet = *mut T; + + fn abi_into(self) -> *mut T { + match self { + Some(b) => Box::into_raw(b), + None => 1 as *mut T, + } + } + + fn null() -> *mut T { + ptr::null_mut() + } +} + +impl AbiInto for *const T { + type AbiRet = *const T; + + fn abi_into(self) -> *const T { + self + } + + fn null() -> *const T { + ptr::null() + } +} + +impl AbiInto for *mut T { + type AbiRet = *mut T; + + fn abi_into(self) -> *mut T { + self + } + + fn null() -> *mut T { + ptr::null_mut() + } +} + +impl AbiInto for usize { + type AbiRet = usize; + + fn abi_into(self) -> usize { + self + 1 + } + + fn null() -> usize { + 0 + } +} diff --git a/autopush_rs/src/server.rs b/autopush_rs/src/server.rs new file mode 100644 index 00000000..3b046f31 --- /dev/null +++ b/autopush_rs/src/server.rs @@ -0,0 +1,599 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::ffi::CStr; +use std::io; +use std::mem; +use std::panic; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +use futures::sync::oneshot; +use futures::task::{self, Task}; +use futures::{Stream, Future, Sink, Async, Poll, AsyncSink, StartSend}; +use libc::c_char; +use serde_json; +use tokio_core::net::{TcpListener, TcpStream}; +use tokio_core::reactor::{Core, Timeout, Handle, Interval}; +use tokio_tungstenite::{accept_async, WebSocketStream}; +use tungstenite::Message; +use uuid::Uuid; + +use client::{Client, RegisteredClient}; +use errors::*; +use protocol::{ClientMessage, ServerMessage, ServerNotification, Notification}; +use queue::{self, AutopushQueue}; +use rt::{self, AutopushError, UnwindGuard}; +use util::{self, RcObject, timeout}; + +#[repr(C)] +pub struct AutopushServer { + inner: UnwindGuard, +} + +struct AutopushServerInner { + opts: Arc, + // Used when shutting down a server + tx: Cell>>, + thread: Cell>>, +} + +#[repr(C)] +pub struct AutopushServerOptions { + pub debug: i32, + pub port: u16, + pub url: *const c_char, + pub ssl_key: *const c_char, + pub ssl_cert: *const c_char, + pub ssl_dh_param: *const c_char, + pub open_handshake_timeout: u32, + pub auto_ping_interval: f64, + pub auto_ping_timeout: f64, + pub max_connections: u32, + pub close_handshake_timeout: u32, + pub json_logging: i32, +} + +pub struct Server { + uaids: RefCell>, + open_connections: Cell, + pub tx: queue::Sender, + pub opts: Arc, + pub handle: Handle, +} + +pub struct ServerOptions { + pub debug: bool, + pub port: u16, + pub url: String, + pub ssl_key: Option, + pub ssl_cert: Option, + pub ssl_dh_param: Option, + pub open_handshake_timeout: Option, + pub auto_ping_interval: Duration, + pub auto_ping_timeout: Duration, + pub max_connections: Option, + pub close_handshake_timeout: Option, +} + +#[no_mangle] +pub extern "C" fn autopush_server_new(opts: *const AutopushServerOptions, + err: &mut AutopushError) + -> *mut AutopushServer +{ + unsafe fn to_s<'a>(ptr: *const c_char) -> Option<&'a str> { + if ptr.is_null() { + return None + } + let s = CStr::from_ptr(ptr).to_str().expect("invalid utf-8"); + if s.is_empty() { + None + } else { + Some(s) + } + } + + unsafe fn ito_dur(seconds: u32) -> Option { + if seconds == 0 { + None + } else { + Some(Duration::new(seconds.into(), 0)) + } + } + + unsafe fn fto_dur(seconds: f64) -> Option { + if seconds == 0.0 { + None + } else { + Some(Duration::new(seconds as u64, + (seconds.fract() * 1_000_000_000.0) as u32)) + } + } + + rt::catch(err, || unsafe { + let opts = &*opts; + + util::init_logging(opts.json_logging != 0); + + let opts = ServerOptions { + debug: opts.debug != 0, + port: opts.port, + url: to_s(opts.url).expect("url must be specified").to_string(), + ssl_key: to_s(opts.ssl_key).map(PathBuf::from), + ssl_cert: to_s(opts.ssl_cert).map(PathBuf::from), + ssl_dh_param: to_s(opts.ssl_dh_param).map(PathBuf::from), + auto_ping_interval: fto_dur(opts.auto_ping_interval) + .expect("ping interval cannot be 0"), + auto_ping_timeout: fto_dur(opts.auto_ping_timeout) + .expect("ping timeout cannot be 0"), + close_handshake_timeout: ito_dur(opts.close_handshake_timeout), + max_connections: if opts.max_connections == 0 { + None + } else { + Some(opts.max_connections) + }, + open_handshake_timeout: ito_dur(opts.open_handshake_timeout), + }; + + Box::new(AutopushServer { + inner: UnwindGuard::new(AutopushServerInner { + opts: Arc::new(opts), + tx: Cell::new(None), + thread: Cell::new(None), + }), + }) + }) +} + +#[no_mangle] +pub extern "C" fn autopush_server_start(srv: *mut AutopushServer, + queue: *mut AutopushQueue, + err: &mut AutopushError) -> i32 { + unsafe { + (*srv).inner.catch(err, |srv| { + let tx = (*queue).tx(); + let (tx, thread) = Server::start(&srv.opts, tx) + .expect("failed to start server"); + srv.tx.set(Some(tx)); + srv.thread.set(Some(thread)); + }) + } +} + +#[no_mangle] +pub extern "C" fn autopush_server_stop(srv: *mut AutopushServer, + err: &mut AutopushError) -> i32 { + unsafe { + (*srv).inner.catch(err, |srv| { + srv.stop().expect("tokio thread panicked"); + }) + } +} + +#[no_mangle] +pub extern "C" fn autopush_server_free(srv: *mut AutopushServer) { + rt::abort_on_panic(|| unsafe { + Box::from_raw(srv); + }) +} + +impl AutopushServerInner { + /// Blocks execution of the calling thread until the helper thread with the + /// tokio reactor has exited. + fn stop(&self) -> Result<()> { + drop(self.tx.take()); + if let Some(thread) = self.thread.take() { + thread.join().map_err(ErrorKind::Thread)?; + } + Ok(()) + } +} + +impl Drop for AutopushServerInner { + fn drop(&mut self) { + drop(self.stop()); + } +} + +impl Server { + /// Creates a new server handle to send to python. + /// + /// This will spawn a new server with the `opts` specified, spinning up a + /// separate thread for the tokio reactor. The returned + /// `AutopushServerInner` is a handle to the spawned thread and can be used + /// to interact with it (e.g. shut it down). + fn start(opts: &Arc, tx: queue::Sender) + -> io::Result<(oneshot::Sender<()>, thread::JoinHandle<()>)> + { + let (donetx, donerx) = oneshot::channel(); + let (inittx, initrx) = oneshot::channel(); + + let opts = opts.clone(); + assert!(opts.ssl_key.is_none(), "ssl not supported"); + assert!(opts.ssl_cert.is_none(), "ssl not supported"); + assert!(opts.ssl_dh_param.is_none(), "ssl not supported"); + + let thread = thread::spawn(move || { + let (srv, mut core) = match Server::new(&opts, tx) { + Ok(core) => { + inittx.send(None).unwrap(); + core + } + Err(e) => return inittx.send(Some(e)).unwrap(), + }; + + // For now during development spin up a dummy HTTP server which is + // used to send notifications to clients. + { + use hyper::server::Http; + + let handle = core.handle(); + let addr = "127.0.0.1:8081".parse().unwrap(); + let push_listener = TcpListener::bind(&addr, &handle).unwrap(); + let proto = Http::new(); + let push_srv = push_listener.incoming().for_each(move |(socket, addr)| { + proto.bind_connection(&handle, socket, addr, + ::http::Push(srv.clone())); + Ok(()) + }); + core.handle().spawn(push_srv.then(|res| { + info!("Http server {:?}", res); + Ok(()) + })); + } + + drop(core.run(donerx)); + }); + + match initrx.wait() { + Ok(Some(e)) => Err(e), + Ok(None) => Ok((donetx, thread)), + Err(_) => panic::resume_unwind(thread.join().unwrap_err()), + } + } + + fn new(opts: &Arc, tx: queue::Sender) + -> io::Result<(Rc, Core)> + { + let core = Core::new()?; + let srv = Rc::new(Server { + opts: opts.clone(), + uaids: RefCell::new(HashMap::new()), + open_connections: Cell::new(0), + handle: core.handle(), + tx: tx, + }); + let addr = format!("127.0.0.1:{}", srv.opts.port); + let ws_listener = TcpListener::bind(&addr.parse().unwrap(), &srv.handle)?; + + assert!(srv.opts.ssl_key.is_none(), "ssl not supported yet"); + assert!(srv.opts.ssl_cert.is_none(), "ssl not supported yet"); + assert!(srv.opts.ssl_dh_param.is_none(), "ssl not supported yet"); + + let handle = core.handle(); + let srv2 = srv.clone(); + let ws_srv = ws_listener.incoming() + .map_err(|e| Error::from(e)) + + .for_each(move |(socket, addr)| { + // Make sure we're not handling too many clients before we start the + // websocket handshake. + let max = srv.opts.max_connections.unwrap_or(u32::max_value()); + if srv.open_connections.get() >= max { + info!("dropping {} as we already have too many open \ + connections", addr); + return Ok(()) + } + srv.open_connections.set(srv.open_connections.get() + 1); + + // TODO: TCP socket options here? + + // Perform the websocket handshake on each connection, but don't let + // it take too long. + let ws = accept_async(socket, None).chain_err(|| "failed to accept client"); + let ws = timeout(ws, srv.opts.open_handshake_timeout, &handle); + + // Once the handshake is done we'll start the main communication + // with the client, managing pings here and deferring to + // `Client` to start driving the internal state machine. + let srv2 = srv.clone(); + let client = ws.and_then(move |ws| { + PingManager::new(&srv2, ws) + .chain_err(|| "failed to make ping handler") + }).flatten(); + + let srv = srv.clone(); + handle.spawn(client.then(move |res| { + srv.open_connections.set(srv.open_connections.get() - 1); + if let Err(e) = res { + let mut error = e.to_string(); + for err in e.iter().skip(1) { + error.push_str("\n"); + error.push_str(&err.to_string()); + } + error!("{}: {}", addr, error); + } + Ok(()) + })); + + Ok(()) + }); + + core.handle().spawn(ws_srv.then(|res| { + debug!("srv res: {:?}", res.map(drop)); + Ok(()) + })); + + Ok((srv2, core)) + } + + /// Informs this server that a new `client` has connected + /// + /// For now just registers internal state by keeping track of the `client`, + /// namely its channel to send notifications back. + pub fn connect_client(&self, client: RegisteredClient) { + debug!("Connecting a client!"); + assert!(self.uaids.borrow_mut().insert(client.uaid, client).is_none()); + } + + /// A notification has come for the uaid + pub fn notify_client(&self, uaid: Uuid, notif: Notification) -> Result<()> { + let mut uaids = self.uaids.borrow_mut(); + if let Some(client) = uaids.get_mut(&uaid) { + debug!("Found a client to deliver a notification to"); + // TODO: Don't unwrap, handle error properly + (&client.tx).send(ServerNotification::Notification(notif)).unwrap(); + info!("Dropped notification in queue"); + return Ok(()); + } + Err("User not connected".into()) + } + + /// The client specified by `uaid` has disconnected. + pub fn disconnet_client(&self, uaid: &Uuid) { + debug!("Disconnecting client!"); + let mut uaids = self.uaids.borrow_mut(); + uaids.remove(uaid).expect("uaid not registered"); + } +} + +impl Drop for Server { + fn drop(&mut self) { + // we're done sending messages, close out the queue + drop(self.tx.send(None)); + } +} + +struct PingManager { + socket: RcObject>>, + ping_interval: Interval, + timeout: TimeoutState, + srv: Rc, + client: CloseState>>>>, +} + +enum TimeoutState { + None, + Ping(Timeout), + Close(Timeout), +} + +enum CloseState { + Exchange(T), + Closing, +} + +impl PingManager { + fn new(srv: &Rc, socket: WebSocketStream) + -> io::Result + { + // The `socket` is itself a sink and a stream, and we've also got a sink + // (`tx`) and a stream (`rx`) to send messages. Half of our job will be + // doing all this proxying: reading messages from `socket` and sending + // them to `tx` while also reading messages from `rx` and sending them + // on `socket`. + // + // Our other job will be to manage the websocket protocol pings going + // out and coming back. The `opts` provided indicate how often we send + // pings and how long we'll wait for the ping to come back before we + // time it out. + // + // To make these tasks easier we start out by throwing the `socket` into + // an `Rc` object. This'll allow us to share it between the ping/pong + // management and message shuffling. + let socket = RcObject::new(WebpushSocket::new(socket)); + Ok(PingManager { + ping_interval: Interval::new(srv.opts.auto_ping_interval, &srv.handle)?, + timeout: TimeoutState::None, + socket: socket.clone(), + client: CloseState::Exchange(Client::new(socket, srv)), + srv: srv.clone(), + }) + } +} + +impl Future for PingManager { + type Item = (); + type Error = Error; + + fn poll(&mut self) -> Poll<(), Error> { + // If it's time for us to send a ping, then queue up a ping to get sent + // and start the clock for that ping to time out. + while let Async::Ready(_) = self.ping_interval.poll()? { + match self.timeout { + TimeoutState::None => {} + _ => continue, + } + self.socket.borrow_mut().ping = true; + let timeout = Timeout::new(self.srv.opts.auto_ping_timeout, &self.srv.handle)?; + self.timeout = TimeoutState::Ping(timeout); + } + + // If the client takes too long to respond to our websocket ping or too + // long to execute the closing handshake then we terminate the whole + // connection. + match self.timeout { + TimeoutState::None => {} + TimeoutState::Close(ref mut timeout) => { + if timeout.poll()?.is_ready() { + if let CloseState::Exchange(ref mut client) = self.client { + client.shutdown(); + } + return Err("close handshake took too long".into()) + } + } + TimeoutState::Ping(ref mut timeout) => { + if timeout.poll()?.is_ready() { + if let CloseState::Exchange(ref mut client) = self.client { + client.shutdown(); + } + return Err("pong not received within timeout".into()) + } + } + } + + // Received pongs will clear our ping timeout, but not the close + // timeout. + if self.socket.borrow_mut().poll_pong().is_ready() { + if let TimeoutState::Ping(_) = self.timeout { + self.timeout = TimeoutState::None; + } + } + + // At this point looks our state of ping management A-OK, so try to + // make progress on our client, and when done with that execute the + // closing handshake. + loop { + match self.client { + CloseState::Exchange(ref mut client) => try_ready!(client.poll()), + CloseState::Closing => return Ok(self.socket.close()?), + } + + self.client = CloseState::Closing; + if let Some(dur) = self.srv.opts.close_handshake_timeout { + let timeout = Timeout::new(dur, &self.srv.handle)?; + self.timeout = TimeoutState::Close(timeout); + } + } + } +} + +// Wrapper struct to take a Sink/Stream of `Message` to a Sink/Stream of +// `ClientMessage` and `ServerMessage`. +struct WebpushSocket { + inner: T, + pong: Pong, + ping: bool, +} + +enum Pong { + None, + Received, + Waiting(Task), +} + +impl WebpushSocket { + fn new(t: T) -> WebpushSocket { + WebpushSocket { + inner: t, + pong: Pong::None, + ping: false, + } + } + + fn poll_pong(&mut self) -> Async<()> { + match mem::replace(&mut self.pong, Pong::None) { + Pong::None => {} + Pong::Received => return Async::Ready(()), + Pong::Waiting(_) => {} + } + self.pong = Pong::Waiting(task::current()); + Async::NotReady + } + + fn send_ping(&mut self) -> Poll<(), Error> + where T: Sink, Error: From + { + if self.ping { + match self.inner.start_send(Message::Ping(Vec::new()))? { + AsyncSink::Ready => self.ping = false, + AsyncSink::NotReady(_) => return Ok(Async::NotReady), + } + } + Ok(Async::Ready(())) + } +} + +impl Stream for WebpushSocket + where T: Stream, + Error: From, +{ + type Item = ClientMessage; + type Error = Error; + + fn poll(&mut self) -> Poll, Error> { + loop { + match try_ready!(self.inner.poll()) { + Some(Message::Text(ref s)) => { + let msg = serde_json::from_str(s).chain_err(|| "invalid json text")?; + return Ok(Some(msg).into()) + } + + Some(Message::Binary(_)) => { + return Err("binary messages not accepted".into()) + } + + // sending a pong is already managed by lower layers, just go to + // the next message + Some(Message::Ping(_)) => {} + + // Wake up tasks waiting for a pong, if any. + Some(Message::Pong(_)) => { + match mem::replace(&mut self.pong, Pong::Received) { + Pong::None => {} + Pong::Received => {} + Pong::Waiting(task) => { + self.pong = Pong::None; + task.notify(); + } + } + } + + None => return Ok(None.into()), + } + } + } +} + +impl Sink for WebpushSocket + where T: Sink, + Error: From, +{ + type SinkItem = ServerMessage; + type SinkError = Error; + + fn start_send(&mut self, msg: ServerMessage) + -> StartSend + { + if self.send_ping()?.is_not_ready() { + return Ok(AsyncSink::NotReady(msg)) + } + let s = serde_json::to_string(&msg).chain_err(|| "failed to serialize")?; + match self.inner.start_send(Message::Text(s))? { + AsyncSink::Ready => Ok(AsyncSink::Ready), + AsyncSink::NotReady(_) => Ok(AsyncSink::NotReady(msg)), + } + } + + fn poll_complete(&mut self) -> Poll<(), Error> { + try_ready!(self.send_ping()); + Ok(self.inner.poll_complete()?) + } + + fn close(&mut self) -> Poll<(), Error> { + try_ready!(self.poll_complete()); + Ok(self.inner.close()?) + } +} diff --git a/autopush_rs/src/util/mod.rs b/autopush_rs/src/util/mod.rs new file mode 100644 index 00000000..92bad25a --- /dev/null +++ b/autopush_rs/src/util/mod.rs @@ -0,0 +1,90 @@ +//! Various small utilities accumulated over time for the WebPush server +use std::env; +use std::time::Duration; +use std::sync::atomic::{ATOMIC_BOOL_INIT, AtomicBool, Ordering}; + +use env_logger; +use futures::future::{Either, Future, IntoFuture}; +use log::LogRecord; +use serde_json; +use tokio_core::reactor::{Handle, Timeout}; + +use errors::*; + +mod send_all; +mod rc; + +pub use self::send_all::MySendAll; +pub use self::rc::RcObject; + +/// Convenience future to time out the resolution of `f` provided within the +/// duration provided. +/// +/// If the `dur` is `None` then the returned future is equivalent to `f` (no +/// timeout) and otherwise the returned future will cancel `f` and resolve to an +/// error if the `dur` timeout elapses before `f` resolves. +pub fn timeout(f: F, dur: Option, handle: &Handle) -> MyFuture + where F: Future + 'static, + F::Error: Into, +{ + let dur = match dur { + Some(dur) => dur, + None => return Box::new(f.map_err(|e| e.into())), + }; + let timeout = Timeout::new(dur, handle).into_future().flatten(); + Box::new(f.select2(timeout).then(|res| { + match res { + Ok(Either::A((item, _timeout))) => Ok(item), + Err(Either::A((e, _timeout))) => Err(e.into()), + Ok(Either::B(((), _item))) => Err("timed out".into()), + Err(Either::B((e, _item))) => Err(e.into()), + } + })) +} + +static LOG_JSON: AtomicBool = ATOMIC_BOOL_INIT; + +pub fn init_logging(json: bool) { + // We only initialize once, so ignore initialization errors related to + // calling this function twice. + let mut builder = env_logger::LogBuilder::new(); + + if env::var("RUST_LOG").is_ok() { + builder.parse(&env::var("RUST_LOG").unwrap()); + } + + builder.target(env_logger::LogTarget::Stdout) + .format(maybe_json_record); + + if builder.init().is_ok() { + LOG_JSON.store(json, Ordering::SeqCst); + } +} + +fn maybe_json_record(record: &LogRecord) -> String { + #[derive(Serialize)] + struct Message<'a> { + msg: String, + level: String, + target: &'a str, + module: &'a str, + file: &'a str, + line: u32, + } + + if LOG_JSON.load(Ordering::SeqCst) { + serde_json::to_string(&Message { + msg: record.args().to_string(), + level: record.level().to_string(), + target: record.target(), + module: record.location().module_path(), + file: record.location().file(), + line: record.location().line(), + }).unwrap() + } else { + format!("{}:{}: {}", + record.level(), + record.location().module_path(), + record.args()) + } +} diff --git a/autopush_rs/src/util/rc.rs b/autopush_rs/src/util/rc.rs new file mode 100644 index 00000000..6855efa0 --- /dev/null +++ b/autopush_rs/src/util/rc.rs @@ -0,0 +1,55 @@ +use std::rc::Rc; +use std::cell::{RefCell, RefMut}; + +use futures::{Stream, Sink, StartSend, Poll}; + +/// Helper object to turn `Rc>` into a `Stream` and `Sink` +/// +/// This is basically just a helper to allow multiple "owning" references to a +/// `T` which is both a `Stream` and a `Sink`. Similar to `Stream::split` in the +/// futures crate, but doesn't actually split it (and allows internal access). +pub struct RcObject(Rc>); + +impl RcObject { + pub fn new(t: T) -> RcObject { + RcObject(Rc::new(RefCell::new(t))) + } + + pub fn borrow_mut(&self) -> RefMut { + self.0.borrow_mut() + } +} + +impl Stream for RcObject { + type Item = T::Item; + type Error = T::Error; + + fn poll(&mut self) -> Poll, T::Error> { + self.0.borrow_mut().poll() + } +} + +impl Sink for RcObject { + type SinkItem = T::SinkItem; + type SinkError = T::SinkError; + + fn start_send(&mut self, msg: T::SinkItem) + -> StartSend + { + self.0.borrow_mut().start_send(msg) + } + + fn poll_complete(&mut self) -> Poll<(), T::SinkError> { + self.0.borrow_mut().poll_complete() + } + + fn close(&mut self) -> Poll<(), T::SinkError> { + self.0.borrow_mut().close() + } +} + +impl Clone for RcObject { + fn clone(&self) -> RcObject { + RcObject(self.0.clone()) + } +} diff --git a/autopush_rs/src/util/send_all.rs b/autopush_rs/src/util/send_all.rs new file mode 100644 index 00000000..0f4e51c6 --- /dev/null +++ b/autopush_rs/src/util/send_all.rs @@ -0,0 +1,83 @@ +use futures::{Stream, Future, Sink, Poll, Async, AsyncSink}; +use futures::stream::Fuse; + +// This is a copy of `Future::forward`, except that it doesn't close the sink +// when it's finished. +pub struct MySendAll { + sink: Option, + stream: Option>, + buffered: Option, +} + +impl MySendAll + where U: Sink, + T: Stream, + T::Error: From, +{ + #[allow(unused)] + pub fn new(t: T, u: U) -> MySendAll { + MySendAll { + sink: Some(u), + stream: Some(t.fuse()), + buffered: None, + } + } + + fn sink_mut(&mut self) -> &mut U { + self.sink.as_mut().take() + .expect("Attempted to poll MySendAll after completion") + } + + fn stream_mut(&mut self) -> &mut Fuse { + self.stream.as_mut().take() + .expect("Attempted to poll MySendAll after completion") + } + + fn take_result(&mut self) -> (T, U) { + let sink = self.sink.take() + .expect("Attempted to poll MySendAll after completion"); + let fuse = self.stream.take() + .expect("Attempted to poll MySendAll after completion"); + (fuse.into_inner(), sink) + } + + fn try_start_send(&mut self, item: T::Item) -> Poll<(), U::SinkError> { + debug_assert!(self.buffered.is_none()); + if let AsyncSink::NotReady(item) = try!(self.sink_mut().start_send(item)) { + self.buffered = Some(item); + return Ok(Async::NotReady) + } + Ok(Async::Ready(())) + } +} + +impl Future for MySendAll + where U: Sink, + T: Stream, + T::Error: From, +{ + type Item = (T, U); + type Error = T::Error; + + fn poll(&mut self) -> Poll<(T, U), T::Error> { + // If we've got an item buffered already, we need to write it to the + // sink before we can do anything else + if let Some(item) = self.buffered.take() { + try_ready!(self.try_start_send(item)) + } + + loop { + match try!(self.stream_mut().poll()) { + Async::Ready(Some(item)) => try_ready!(self.try_start_send(item)), + Async::Ready(None) => { + try_ready!(self.sink_mut().poll_complete()); + return Ok(Async::Ready(self.take_result())) + } + Async::NotReady => { + try_ready!(self.sink_mut().poll_complete()); + return Ok(Async::NotReady) + } + } + } + } +} diff --git a/requirements.txt b/requirements.txt index d9efb77a..5b6be0a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -e git+https://github.com/habnabit/txstatsd.git@157ef85fbdeafe23865c7c4e176237ffcb3c3f1f#egg=txStatsD-master +-e git+https://github.com/mitsuhiko/snaek.git@2b14b8b010a9486af0f298b4ad4c73dc1ceff9d6#egg=snaek apns==2.0.1 asn1crypto==0.22.0 # via cryptography attrs==17.2.0 diff --git a/setup.py b/setup.py index 2f390a51..de5f9cfe 100644 --- a/setup.py +++ b/setup.py @@ -38,11 +38,17 @@ [console_scripts] autopush = autopush.main:ConnectionApplication.main autoendpoint = autopush.main:EndpointApplication.main + autopush_rs = autopush.main:RustConnectionApplication.main autokey = autokey:main endpoint_diagnostic = autopush.diagnostic_cli:run_endpoint_diagnostic_cli drop_users = autopush.scripts.drop_user:drop_users [nose.plugins] object-tracker = autopush.noseplugin:ObjectTracker """, + setup_requires=['snaek'], + install_requires=['snaek'], + snaek_rust_modules=[ + ('autopush_rs._native', 'autopush_rs/'), + ], **extra_options ) diff --git a/states.dot b/states.dot new file mode 100644 index 00000000..75097cf4 --- /dev/null +++ b/states.dot @@ -0,0 +1,74 @@ +digraph g{ + ranksep="1.7 equally" + nodesep="0.9 equally" + + START -> CONNECTION_ESTABLISHED [label="on_connect"]; + + subgraph level0 { + DISCONNECT; + } + + subgraph cluster_unauthenticated { + label = "UNAUTHENTICATED"; + PROCESS_HELLO [ color = "red", penwidth = 2]; + CONNECTION_ESTABLISHED; + PROCESS_HELLO; + } + + subgraph cluster_authenticated { + label = "AUTHENTICATED"; + AWAIT_COMMAND [ color = "blue"]; + RUN_COMMAND; + WAIT_FOR_ACKS [ color = "blue"]; + DELIVER_NOTIFICATIONS; + INC_STORAGE_POSITION [ color = "red", penwidth = 2]; + STORE_DIRECT_MESSAGES [ color = "red", penwidth = 2]; + CHECK_STORAGE [ color = "red", penwidth = 2]; + RUN_COMMAND [ color = "red", penwidth = 2]; + MIGRATE_COMMAND [ color = "red", penwidth = 2]; + DROP_COMMAND [ color = "red", penwidth = 2]; + DELETE_MESSAGE_COMMAND [ color = "red", penwidth = 2]; + } + + CHECK_STORAGE -> DELIVER_NOTIIFCATIONS [label="messages \n Cond:include_topic_flag"]; + CHECK_STORAGE -> DELIVER_NOTIFICATIONS [label="messages \n Set:inc_storage_flag"]; + CHECK_STORAGE -> DROP_COMMAND [label="P1 empty\n Cond:reset_uaid_flag"]; + CHECK_STORAGE -> MIGRATE_COMMAND [label="P2 empty\n Cond:rotate_message_table_flag\n UnSet:check_flag"]; + CHECK_STORAGE -> AWAIT_COMMAND [label="P3 empty\n UnSet:check_flag"]; + + MIGRATE_COMMAND -> AWAIT_COMMAND [label="UnSet:(rotate_message_table_flag,inc_storage_flag,include_topic_flag)"]; + DROP_COMMAND -> DISCONNECT; + + DELIVER_NOTIFICATIONS -> WAIT_FOR_ACKS; + + DELETE_MESSAGE_COMMAND -> WAIT_FOR_ACKS [label="Cond:acks remaining"]; + DELETE_MESSAGE_COMMAND -> CHECK_STORAGE [label="Cond:all acked"]; + + WAIT_FOR_ACKS -> DELETE_MESSAGE_COMMAND [label="topic ack"]; + WAIT_FOR_ACKS -> INC_STORAGE_POSITION [label="all acked \n Cond:check_flag && Cond:inc_storage_flag"]; + WAIT_FOR_ACKS -> CHECK_STORAGE [label="all acked \n Cond:check_flag"]; + WAIT_FOR_ACKS -> AWAIT_COMMAND [label="all acked \n Cond:NO_FLAGS"]; + WAIT_FOR_ACKS -> RUN_COMMAND [label="reg or unreg"]; + WAIT_FOR_ACKS -> DISCONNECT [label="connection drop"]; + WAIT_FOR_ACKS -> STORE_DIRECT_MESSAGES [label="connection drop \n Cond:direct unacked messages"]; + + INC_STORAGE_POSITION -> CHECK_STORAGE [label="UnSet:inc_storage_flag"]; + INC_STORAGE_POSITION -> STORE_DIRECT_MESSAGES [label="connection drop \n Cond:direct unacked messages"]; + INC_STORAGE_POSITION -> DISCONNECT [label="connection drop"]; + + AWAIT_COMMAND -> CHECK_STORAGE [label="endpoint wants full check \nSet:(include_topic,check_flag)", fontcolor="darkgreen", color="green"]; + AWAIT_COMMAND -> RUN_COMMAND [label="reg or unreg"]; + AWAIT_COMMAND -> DISCONNECT [label="connection drop"]; + AWAIT_COMMAND -> DELIVER_NOTIFICATIONS [label="direct message", fontcolor="darkgreen", color="green"]; + + STORE_DIRECT_MESSAGES -> DISCONNECT; + + RUN_COMMAND -> AWAIT_COMMAND [label="send response"]; + RUN_COMMAND -> WAIT_FOR_ACKS [label="send response \n Cond:acks remaining"]; + + CONNECTION_ESTABLISHED -> PROCESS_HELLO [label="hello"]; + CONNECTION_ESTABLISHED -> DISCONNECT [label="ELSE"]; + + PROCESS_HELLO -> CHECK_STORAGE [label="authenticate success \n Set:(include_topic,check_flag)"]; + PROCESS_HELLO -> AWAIT_COMMAND [label="ELSE"]; +} diff --git a/test-requirements.txt b/test-requirements.txt index fa949634..71c5bfb3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,6 +1,7 @@ -r requirements.txt coverage ecdsa==0.13 +factory_boy==2.8.1 flake8==3.3.0 funcsigs==1.0.2 mock>=1.0.1 diff --git a/tox.ini b/tox.ini index 629eaf6c..30f56570 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ setenv = with_gmp=no [testenv:flake8] commands = flake8 autopush -deps = flake8 +deps += flake8 [testenv:py36-mypy] commands = mypy autopush