config.yaml

# Cryptostore sample config file

# Redis or Kafka are required. They are used to batch updates from cryptofeed and the storage medium of choice
#
# del_after_read: (redis only) toggles the removal of data from redis after it has been processed with cryptostore.
# retention_time: (redis only) if data removal is enabled (via del_after_read) will allow retention of data in redis for N seconds.
# socket: (redis only) allows redis connections via a unix domain socket
# start_flush: toggles if redis/kafka should be flushed at the start. Primarily for debugging, it will flush ALL of redis/kafka
cache: redis

kafka:
    # ip/port are for the bootstrap server
    ip: '127.0.0.1'
    port: 9092
    start_flush: true
redis:
    ip: '127.0.0.1'
    port: 6379
    socket: null
    del_after_read: true
    retention_time: null
    start_flush: true

# Data sources and data types configured under exchanges. Exchange names follow the naming scheme in cryptofeed (they
# must be capitalized) and only exchanges supported by cryptofeed are supported.
# data types follow cryptofeed definitions, see defines.py in cryptofeed for more details, common ones are
# trades, l2_book, l3_book, funding, ticker, and open_interest
# Trading pairs for all exchanges (except BitMEX) follow the currency-quote format
#
# max_depth controls the size of the book to return. The top N levels will be returned, only when those N levels
# have changed.
# book_delta enables book deltas (snapshot, then only deltas are delivered). Snapshops are delivered
# every book_interval updates. book_interval defaults to 1000 if not specified
#
# Retries are the number of times the connection to the exchange will be retried before exiting. -1 is infinity.
# Using a retry of -1 with a bug in your config can lead to bans by exchanges
#
# Channel timeouts are channel specific and control how long a channel can go with no messages before declaring the connection
# dead. Cryptofeed will restart the connection if no message is received in this time period. Default is 120 seconds. -1 means no timeout.

exchanges:
    BITMEX:
        channel_timeouts:
            l2_book: 30
            trades: 120
            ticker: 120
            funding: -1
        retries: -1
        l2_book:
            symbols: [XBTUSD]
            max_depth: 10
            book_delta: true
            book_interval: 100000
        trades: [XBTUSD]
        ticker: [XBTUSD]
        funding: [XBTUSD]
    COINBASE:
        retries: -1
        l3_book:
            symbols: [BTC-USD]
            book_delta: true
            book_interval: 100000
        trades: [BTC-USD, ETH-USD, ETH-BTC]
        ticker: [BTC-USD]

# Where to store the data. Currently arctic, influx, elastic, and parquet are supported. More than one can be enabled
storage: [arctic, influx]

# Configurable passthrough for data - data will be sent in realtime (no aggregation in redis).
# To disable, remove
pass_through:
    type: zmq
    host: '127.0.0.1'
    port: 5678


elastic:
    host: 'http://127.0.0.1:9200'
    user: null
    token: null
    shards: 10
    replicas: 0
    refresh_interval: '30s'

influx:
    host: 'http://127.0.0.1:8086'
    db: example
    create: true

# Parquet specific options. Parquet will default to storing the data on disk unless these are specified
parquet:
    # if storing the data to an external source (like S3) toggle this to enable the removal of the local file after
    # writing to external store
    del_file: true

    S3:
        # endpoint allows you to override the write endpoint and write to other provicers that have the same API interface
        # If key_id/secret are NULL boto will default to using ENV vars or credentials file
        # prefix: a prefix to append to the default data path
        endpoint: null
        key_id: null
        secret: null
        bucket: null
        prefix: null
    GCS:
        # path to service account key, if null will default to using env vars or auth tokens
        # on GCE node
        # prefix: a prefix to append to the default data path
        service_account: null
        bucket: null
        prefix: null

# arctic specific configuration options - the mongo URL
arctic: mongodb://127.0.0.1

# Data batching window, in seconds, or optionally a time interval: M(inutely), H(ourly), D(aily).
# These intervals will require that you have enough memory available to redis (and python) to hold this amount of data.
# String intervals can be combined with numbers, eg 2H is 2 hours, 5M is 5 minutes, etc.
storage_interval: 60


# Cryptostore Plugin Interface
plugins:
    backfill:
        # The import path, and class name
        # from cryptostore.plugin.backfill import Backfill
        module: [cryptostore.plugin.backfill, Backfill]
        # Config path, config can be added to this file, or in a separate config
        # backfill does not use a dynamic config, so its fine to put its config here
        config: config.yaml

# If configured, backfill will run in a separate process and
# backfill trade data from earliest data (in storage) up to `start` (inclusive).
backfill:
    COINBASE:
        BTC-USD:
            start: '2017-01-01'