Logging, instant notification, ...

[femtotrader]

Logging, instant notification are necessary for live trading and paper trading.

see https://github.com/mhallsmoore/qstrader/issues/120

We can use

• standard logging module https://docs.python.org/3/library/logging.html
• logbook http://logbook.readthedocs.io/

and handlers for

• email notification (SMTP handler)
• instant notification PushOver https://pushover.net/
• Twilio https://www.twilio.com/ (see https://github.com/twilio/twilio-python )
• Slack integration https://slack.com/
• ...

For critical messages, SMS may also be considered BulkSMS is a quite inexpensive solution http://developer.bulksms.com/eapi/code-samples/python/send_sms/ with a REST API or a mail to SMS gateway

Logging as a service (LaaS) may also be considered: Loggly, logsene...

[femtotrader]

Some code that could help

PushOver notification

pushover_notifier.py

import json
from compat import urlencode, http_client

"""
https://pushover.net/
https://api.pushover.net/1/messages.json

https://pushover.net/api
"""

class PushoverException(Exception):
    pass

class PushoverNotifier:

    _HOST = "api.pushover.net"

    def __init__(self, user_key, api_token, title='default', priority=0,
                 max_title_len=100, max_message_len=512):
        self.user_key = user_key
        self.api_token = api_token
        self.title = title
        if priority not in [-2, -1, 0, 1]:
            priority = 0
        self.priority = priority
        self.max_title_len = max_title_len
        self.max_message_len = max_message_len

    def __send(self, message_text, title, priority):
        endpoint = "/1/messages.json"

        params = {
            "user": self.user_key,
            "token": self.api_token,
            "message": message_text,
            "title": title,
            "priority": priority
        }

        body = urlencode(params)
        con = http_client.HTTPSConnection(self._HOST)
        con.request('POST', endpoint, body=body)
        response = con.getresponse()
        data = json.loads(response.read().decode())
        con.close()

        if data['status'] != 1:
            raise PushoverException(data)

    def _crop(self, msg, max_len):
        if max_len is not None and max_len > 0 and len(msg) > max_len:
            return "%s..." % (msg[:max_len-3],)
        else:
            return msg

    def send(self, message_text="Hello", title=None, priority=None):
        if title is None:
            title = self.title

        if priority is None:
            priority = self.priority

        title = self._crop(title, self.max_title_len)
        message_text = self._crop(message_text, self.max_message_len)

        self.__send(message_text, title, priority)

class PushoverTestNotifier(PushoverNotifier):
    def __send(self, message_text, title):
        print("Sending notification to {user_key}".format(user_key=self.user_key))
        print("=" * 30)
        print(message_text)

config.py

#!/usr/bin/env python

CONFIG = {
    "USER_KEY": "YOUR_USER_KEY",
    "API_TOKEN": "YOUR_API_TOKEN",
    "TITLE": "pushover_cli",
    "PRIORITY": 0
}

and for testing purpose a CLI pushover_cli.py

#!/usr/bin/env python

import argparse
from config import CONFIG
from pushover_notifier import PushoverNotifier, PushoverTestNotifier

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--user_key', action='store', help='user key', default=CONFIG["USER_KEY"])
    parser.add_argument('--api_token', action='store', help='API token', default=CONFIG["API_TOKEN"])
    parser.add_argument('--title', action='store', help='title', default=CONFIG["TITLE"])
    parser.add_argument('--priority', action='store', help='title', default=CONFIG["PRIORITY"])
    parser.add_argument('--message', action='store', help='message text', default='Hello')
    parser.add_argument('--test', action='store_true', help='enable test mode', default=False)
    args = parser.parse_args()

    if not args.test:
        push_notifier = PushoverNotifier(args.user_key, args.api_token, args.title, int(args.priority))
    else:
        push_notifier = PushoverTestNotifier(args.user_key, args.api_token, args.title, int(args.priority))

    push_notifier.send(args.message)
    print("Message sent")

if __name__ == "__main__":
    main()

SMS notification (using REST API of BulkSMS)

sms_notifier.py

#!/usr/bin/env python

# import requests
import time
from enum import Enum
from compat import urlencode, http_client

"""
http://www.bulksms.com/int/w/eapi-sms-gateway.htm
http://www.bulksms.com/int/docs/eapi/submission/send_sms/
"""

ActionWhenLong = Enum("ActionWhenLong", "shorten multiple")

class SmsNotifierException(Exception):
    pass

class SmsNotifier:

    _DEFAULT_HOST = "bulksms.vsms.net:5567"

    def __init__(self, msisdn, username, password, host=_DEFAULT_HOST, allowLongMessage=False, actionWhenLong=ActionWhenLong.shorten):
        self.msisdn = msisdn

        self.host = host

        self.username = username
        self.password = password

        self.allowLongMessage = allowLongMessage
        self.actionWhenLong = actionWhenLong

    def __send(self, message_text, msisdn):
        endpoint = "/eapi/submission/send_sms/2/2.0"

        params = {
            "username": self.username,
            "password": self.password,
            "message": message_text,
            "msisdn": msisdn
        }

        con = http_client.HTTPConnection(self.host)
        con.request('GET', endpoint + "?" + urlencode(params))
        response = con.getresponse()
        s = response.read().decode()
        con.close()

        statusCode, statusString, Id = s.split('|')
        if statusCode != '0':
            raise(SmsNotifierException("Error: {statusCode}: {statusString}".format(statusCode=statusCode, statusString=statusString)))

    def send(self, message_text=None, msisdn=None, allowLongMessage=None, actionWhenLong=None):
        if message_text is None:
            raise(SmsNotifierException("No SMS sent because no message was given"))

        if msisdn is None:
            msisdn = self.msisdn

        if allowLongMessage is None:
            allowLongMessage = self.allowLongMessage

        if actionWhenLong is None:
            actionWhenLong = self.actionWhenLong

        max_size = 160
        size = len(message_text)
        if size > max_size:
            if not allowLongMessage:
                raise(SmsNotifierException("Message body too long - max 160 character and allowLongMessage was set to False"))
            else:
                if actionWhenLong == ActionWhenLong.shorten:
                    self.__send(message_text[0:max_size], msisdn)
                elif actionWhenLong == ActionWhenLong.multiple:
                    id_format = "{id:02d}/{N:02d} "
                    id_size = len(id_format.format(id=1, N=1))
                    N = (size + max_size - 1) / max_size
                    print("{N} messages will be sent".format(N=N))
                    for i in range(N):
                        if i != 0:
                            time.sleep(10)
                        self.__send(id_format.format(id=i+1, N=N)+message_text[i*(max_size-id_size):(i+1)*(max_size-id_size)], msisdn)
                else:  # reject
                    raise(SmsNotifierException("actionWhenLong = '%s' but it should be in %s" % (actionWhenLong, ActionWhenLong)))
        else:
            self.__send(message_text, msisdn)

config.py

#!/usr/bin/env python

CONFIG = {
    "MSISDN": "+xyzxyzxyzxy",
    "USERNAME": "YOURUSERNAME",
    "PASSWORD": "YOURPASSWORD"
}

and a CLI for testing purpose sms_cli.py

#!/usr/bin/env python

import argparse
from config import CONFIG
from sms_notifier import SmsNotifier

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--msisdn', action='store', help='msisdn', default=CONFIG["MSISDN"])
    parser.add_argument('--username', action='store', help='username', default=CONFIG["USERNAME"])
    parser.add_argument('--password', action='store', help='password', default=CONFIG["PASSWORD"])
    parser.add_argument('--message', action='store', help='message text')
    parser.add_argument('--to', action='store', help='send to')
    parser.add_argument('--allowLongMessage', action='store_true', help='use this flag to allow long message')
    parser.add_argument('--actionWhenLong', action='store', help='use this flag to set was action to do when long message are sent')
    args = parser.parse_args()

    sms_notifier = SmsNotifier(args.msisdn, args.username, args.password)

    sms_notifier.send(args.message, args.to, args.allowLongMessage, args.actionWhenLong)

if __name__ == "__main__":
    main()

We just need to make a custom handler for Logbook (or for python standard logging module) and add logging into qstrader.

[femtotrader]

Logbook supports PushOver "out of the box"

from config import CONFIG

import sys

from logbook import Logger, StreamHandler
from logbook.notifiers import PushoverHandler

log = Logger('LogbookExample')

StreamHandler(sys.stdout).push_application()

log_handler = PushoverHandler(apikey=CONFIG["API_TOKEN"], userkey=CONFIG["USER_KEY"], bubble=True)
log_handler.push_application()

def main():
    log.critical('THIS IS A CRITICAL MESSAGE')
    log.error('This is an error message')
    log.warning('This is a warning message')
    log.info('This is an info message')
    log.debug('This is a debug message')

if __name__ == "__main__":
    main()

[femtotrader]

Adding PushOver and SMSBulk as log handler with standard logging Python module, is possible but we need to create our own handlers (that's an easy task anyway).

PushOver

pushover_handler.py

#!/usr/bin/env python

from logging import Handler
from pushover_notifier import PushoverNotifier

class PushoverHandler(Handler):
    def __init__(self, user_key, api_token, title='default', priority=0):
        Handler.__init__(self)
        self.notifier = PushoverNotifier(user_key, api_token, title, priority)

    def emit(self, record):
        try:
            msg = self.format(record)
            self.notifier.send(msg)
        except Exception:
            self.handleError(record)

we can than use it like in logging_example.py

from config import CONFIG

import logging
from pushover_handler import PushoverHandler

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

handler = PushoverHandler(CONFIG["USER_KEY"], CONFIG["API_TOKEN"], CONFIG["TITLE"], CONFIG["PRIORITY"])
formatter = logging.Formatter('%(levelname)-8s %(message)s')
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)  # only log to PushOver message with level >= INFO
logger.addHandler(handler)

def main():
    logger.debug('This is a debug message')
    logger.info('This is an info message')
    logger.warning('This is a warning message')
    logger.error('This is an error message')
    logger.critical('THIS IS A CRITICAL MESSAGE')

if __name__ == "__main__":
    main()

So we can log to console any message (including DEBUG message) but only log to PushOver message with level >= INFO

SMSBulk

Same with SMSBulk we need a handler defined in sms_handler.py

#!/usr/bin/env python

from logging import Handler
from sms_notifier import SmsNotifier, ActionWhenLong

class SmsHandler(Handler):
    def __init__(self, msisdn, username, password, host=SmsNotifier._DEFAULT_HOST,
                 allowLongMessage=False, actionWhenLong=ActionWhenLong.shorten):
        Handler.__init__(self)
        self.notifier = SmsNotifier(msisdn, username, password, host,
                                    allowLongMessage, actionWhenLong)

    def emit(self, record):
        try:
            msg = self.format(record)
            self.notifier.send(msg)
        except Exception:
            self.handleError(record)

and we can use it like in logging_example.py

from config import CONFIG

import logging
from sms_handler import SmsHandler

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

handler = SmsHandler(CONFIG["MSISDN"], CONFIG["USERNAME"], CONFIG["PASSWORD"])
formatter = logging.Formatter('%(levelname)-8s %(message)s')
handler.setFormatter(formatter)
handler.setLevel(logging.CRITICAL)  # only send SMS for message with level >= CRITICAL
logger.addHandler(handler)

def main():
    logger.debug('This is a debug message')
    logger.info('This is an info message')
    logger.warning('This is a warning message')
    logger.error('This is an error message')
    logger.critical('THIS IS A CRITICAL MESSAGE')

if __name__ == "__main__":
    main()

in this example only CRITICAL message are sent using SMS.

[femtotrader]

Slack

Some code for Slack integration with logging (standard module) using slacker

slack_handler.py

#!/usr/bin/env python

import logging
logging.getLogger("requests").setLevel(logging.WARNING)
from logging import Handler
from slacker import Slacker

class SlackHandler(Handler):
    _DEFAULT_CHANNEL = '#general'

    def __init__(self, token, incoming_webhook_url=None, timeout=10, channel=_DEFAULT_CHANNEL):
        Handler.__init__(self)
        self.slack = Slacker(token, incoming_webhook_url, timeout)
        self.channel = channel

    def emit(self, record):
        try:
            msg = self.format(record)
            self.slack.chat.post_message(self.channel, msg)
        except Exception:
            self.handleError(record)

and usage in logging_example.py

from config import CONFIG

import logging

from slack_handler import SlackHandler

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)

handler = SlackHandler(CONFIG["API_TOKEN"])
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)  # only log to Slack message with level >= INFO
logger.addHandler(handler)

def main():
    logger.critical('THIS IS A CRITICAL MESSAGE')
    logger.error('This is an error message')
    logger.warning('This is a warning message')
    logger.info('This is an info message')
    logger.debug('This is a debug message')

if __name__ == "__main__":
    main()