Merge branch 'develop' into align_utils

docs/utils.md

@@ -54,3 +54,73 @@ Timeframes available for the exchange `binance`: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 4
 ```
 $ for i in `freqtrade list-exchanges -1`; do freqtrade list-timeframes --exchange $i; done
 ```
+
+## List pairs/list markets
+
+The `list-pairs` and `list-markets` subcommands allow to see the pairs/markets available on exchange.
+
+Pairs are markets with the '/' character between the base currency part and the quote currency part in the market symbol.
+For example, in the 'ETH/BTC' pair 'ETH' is the base currency, while 'BTC' is the quote currency.
+
+For pairs traded by Freqtrade the pair quote currency is defined by the value of the `stake_currency` configuration setting.
+
+You can print info about any pair/market with these subcommands - and you can filter output by quote-currency using `--quote BTC`, or by base-currency using `--base ETH` options correspondingly.
+
+These subcommands have same usage and same set of available options:
+
+```
+usage: freqtrade list-markets [-h] [--exchange EXCHANGE] [--print-list]
+                              [--print-json] [-1] [--print-csv]
+                              [--base BASE_CURRENCY [BASE_CURRENCY ...]]
+                              [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]]
+                              [-a]
+
+usage: freqtrade list-pairs [-h] [--exchange EXCHANGE] [--print-list]
+                            [--print-json] [-1] [--print-csv]
+                            [--base BASE_CURRENCY [BASE_CURRENCY ...]]
+                            [--quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]] [-a]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --exchange EXCHANGE   Exchange name (default: `bittrex`). Only valid if no
+                        config is provided.
+  --print-list          Print list of pairs or market symbols. By default data
+                        is printed in the tabular format.
+  --print-json          Print list of pairs or market symbols in JSON format.
+  -1, --one-column      Print output in one column.
+  --print-csv           Print exchange pair or market data in the csv format.
+  --base BASE_CURRENCY [BASE_CURRENCY ...]
+                        Specify base currency(-ies). Space-separated list.
+  --quote QUOTE_CURRENCY [QUOTE_CURRENCY ...]
+                        Specify quote currency(-ies). Space-separated list.
+  -a, --all             Print all pairs or market symbols. By default only
+                        active ones are shown.
+```
+
+By default, only active pairs/markets are shown. Active pairs/markets are those that can currently be traded
+on the exchange. The see the list of all pairs/markets (not only the active ones), use the `-a`/`-all` option.
+
+Pairs/markets are sorted by its symbol string in the printed output.
+
+### Examples
+
+* Print the list of active pairs with quote currency USD on exchange, specified in the default
+configuration file (i.e. pairs on the "Bittrex" exchange) in JSON format:
+
+```
+$ freqtrade list-pairs --quote USD --print-json
+```
+
+* Print the list of all pairs on the exchange, specified in the `config_binance.json` configuration file
+(i.e. on the "Binance" exchange) with base currencies BTC or ETH and quote currencies USDT or USD, as the
+human-readable list with summary:
+
+```
+$ freqtrade -c config_binance.json list-pairs --all --base BTC ETH --quote USDT USD --print-list
+```
+
+* Print all markets on exchange "Kraken", in the tabular format:
+
+```
+$ freqtrade list-markets --exchange kraken --all
+```

freqtrade/configuration/arguments.py

@@ -2,6 +2,7 @@
 This module contains the argument manager class
 """
 import argparse
+from functools import partial
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -33,6 +34,9 @@
 
 ARGS_LIST_TIMEFRAMES = ["exchange", "print_one_column"]
 
+ARGS_LIST_PAIRS = ["exchange", "print_list", "list_pairs_print_json", "print_one_column",
+                   "print_csv", "base_currencies", "quote_currencies", "list_pairs_all"]
+
 ARGS_CREATE_USERDIR = ["user_data_dir"]
 
 ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "download_trades", "exchange",
@@ -44,7 +48,8 @@
 ARGS_PLOT_PROFIT = ["pairs", "timerange", "export", "exportfilename", "db_url",
                     "trade_source", "ticker_interval"]
 
-NO_CONF_REQURIED = ["download-data", "list-timeframes", "plot-dataframe", "plot-profit"]
+NO_CONF_REQURIED = ["download-data", "list-timeframes", "list-markets", "list-pairs",
+                    "plot-dataframe", "plot-profit"]
 
 NO_CONF_ALLOWED = ["create-userdir", "list-exchanges"]
 
@@ -107,7 +112,8 @@ def _build_subcommands(self) -> None:
         """
         from freqtrade.optimize import start_backtesting, start_hyperopt, start_edge
         from freqtrade.utils import (start_create_userdir, start_download_data,
-                                     start_list_exchanges, start_list_timeframes)
+                                     start_list_exchanges, start_list_timeframes,
+                                     start_list_markets)
 
         subparsers = self.parser.add_subparsers(dest='subparser')
 
@@ -148,6 +154,22 @@ def _build_subcommands(self) -> None:
         list_timeframes_cmd.set_defaults(func=start_list_timeframes)
         self._build_args(optionlist=ARGS_LIST_TIMEFRAMES, parser=list_timeframes_cmd)
 
+        # Add list-markets subcommand
+        list_markets_cmd = subparsers.add_parser(
+            'list-markets',
+            help='Print markets on exchange.'
+        )
+        list_markets_cmd.set_defaults(func=partial(start_list_markets, pairs_only=False))
+        self._build_args(optionlist=ARGS_LIST_PAIRS, parser=list_markets_cmd)
+
+        # Add list-pairs subcommand
+        list_pairs_cmd = subparsers.add_parser(
+            'list-pairs',
+            help='Print pairs on exchange.'
+        )
+        list_pairs_cmd.set_defaults(func=partial(start_list_markets, pairs_only=True))
+        self._build_args(optionlist=ARGS_LIST_PAIRS, parser=list_pairs_cmd)
+
         # Add download-data subcommand
         download_data_cmd = subparsers.add_parser(
             'download-data',

freqtrade/configuration/cli_options.py

@@ -252,6 +252,42 @@ def __init__(self, *args, **kwargs):
         help='Print all exchanges known to the ccxt library.',
         action='store_true',
     ),
+    # List pairs / markets
+    "list_pairs_all": Arg(
+        '-a', '--all',
+        help='Print all pairs or market symbols. By default only active '
+             'ones are shown.',
+        action='store_true',
+    ),
+    "print_list": Arg(
+        '--print-list',
+        help='Print list of pairs or market symbols. By default data is '
+             'printed in the tabular format.',
+        action='store_true',
+    ),
+    "list_pairs_print_json": Arg(
+        '--print-json',
+        help='Print list of pairs or market symbols in JSON format.',
+        action='store_true',
+        default=False,
+    ),
+    "print_csv": Arg(
+        '--print-csv',
+        help='Print exchange pair or market data in the csv format.',
+        action='store_true',
+    ),
+    "quote_currencies": Arg(
+        '--quote',
+        help='Specify quote currency(-ies). Space-separated list.',
+        nargs='+',
+        metavar='QUOTE_CURRENCY',
+    ),
+    "base_currencies": Arg(
+        '--base',
+        help='Specify base currency(-ies). Space-separated list.',
+        nargs='+',
+        metavar='BASE_CURRENCY',
+    ),
     # Script options
     "pairs": Arg(
         '-p', '--pairs',

freqtrade/exchange/__init__.py

@@ -10,5 +10,7 @@
                                          timeframe_to_msecs,
                                          timeframe_to_next_date,
                                          timeframe_to_prev_date)
+from freqtrade.exchange.exchange import (market_is_active,  # noqa: F401
+                                         symbol_is_pair)
 from freqtrade.exchange.kraken import Kraken  # noqa: F401
 from freqtrade.exchange.binance import Binance  # noqa: F401

freqtrade/exchange/exchange.py

@@ -22,6 +22,7 @@
 from freqtrade.data.converter import parse_ticker_dataframe
 from freqtrade.misc import deep_merge_dicts
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -216,20 +217,22 @@ def __init__(self, config: dict, validate: bool = True) -> None:
 
         logger.info('Using Exchange "%s"', self.name)
 
-        # Check if timeframe is available
-        self.validate_timeframes(config.get('ticker_interval'))
-
-        # Converts the interval provided in minutes in config to seconds
-        self.markets_refresh_interval: int = exchange_config.get(
-            "markets_refresh_interval", 60) * 60
         if validate:
+            # Check if timeframe is available
+            self.validate_timeframes(config.get('ticker_interval'))
+
             # Initial markets load
             self._load_markets()
+
             # Check if all pairs are available
             self.validate_pairs(config['exchange']['pair_whitelist'])
             self.validate_ordertypes(config.get('order_types', {}))
             self.validate_order_time_in_force(config.get('order_time_in_force', {}))
 
+        # Converts the interval provided in minutes in config to seconds
+        self.markets_refresh_interval: int = exchange_config.get(
+            "markets_refresh_interval", 60) * 60
+
     def __del__(self):
         """
         Destructor - clean up async stuff
@@ -293,6 +296,28 @@ def markets(self) -> Dict:
             self._load_markets()
         return self._api.markets
 
+    def get_markets(self, base_currencies: List[str] = None, quote_currencies: List[str] = None,
+                    pairs_only: bool = False, active_only: bool = False) -> Dict:
+        """
+        Return exchange ccxt markets, filtered out by base currency and quote currency
+        if this was requested in parameters.
+
+        TODO: consider moving it to the Dataprovider
+        """
+        markets = self.markets
+        if not markets:
+            raise OperationalException("Markets were not loaded.")
+
+        if base_currencies:
+            markets = {k: v for k, v in markets.items() if v['base'] in base_currencies}
+        if quote_currencies:
+            markets = {k: v for k, v in markets.items() if v['quote'] in quote_currencies}
+        if pairs_only:
+            markets = {k: v for k, v in markets.items() if symbol_is_pair(v['symbol'])}
+        if active_only:
+            markets = {k: v for k, v in markets.items() if market_is_active(v)}
+        return markets
+
     def klines(self, pair_interval: Tuple[str, str], copy=True) -> DataFrame:
         if pair_interval in self._klines:
             return self._klines[pair_interval].copy() if copy else self._klines[pair_interval]
@@ -1074,3 +1099,27 @@ def timeframe_to_next_date(timeframe: str, date: datetime = None) -> datetime:
     new_timestamp = ccxt.Exchange.round_timeframe(timeframe, date.timestamp() * 1000,
                                                   ROUND_UP) // 1000
     return datetime.fromtimestamp(new_timestamp, tz=timezone.utc)
+
+
+def symbol_is_pair(market_symbol: str, base_currency: str = None, quote_currency: str = None):
+    """
+    Check if the market symbol is a pair, i.e. that its symbol consists of the base currency and the
+    quote currency separated by '/' character. If base_currency and/or quote_currency is passed,
+    it also checks that the symbol contains appropriate base and/or quote currency part before
+    and after the separating character correspondingly.
+    """
+    symbol_parts = market_symbol.split('/')
+    return (len(symbol_parts) == 2 and
+            (symbol_parts[0] == base_currency if base_currency else len(symbol_parts[0]) > 0) and
+            (symbol_parts[1] == quote_currency if quote_currency else len(symbol_parts[1]) > 0))
+
+
+def market_is_active(market):
+    """
+    Return True if the market is active.
+    """
+    # "It's active, if the active flag isn't explicitly set to false. If it's missing or
+    # true then it's true. If it's undefined, then it's most likely true, but not 100% )"
+    # See https://github.com/ccxt/ccxt/issues/4874,
+    # https://github.com/ccxt/ccxt/issues/4075#issuecomment-434760520
+    return market.get('active', True) is not False

freqtrade/misc.py

@@ -123,3 +123,7 @@ def round_dict(d, n):
     Rounds float values in the dict to n digits after the decimal point.
     """
     return {k: (round(v, n) if isinstance(v, float) else v) for k, v in d.items()}
+
+
+def plural(num, singular: str, plural: str = None) -> str:
+    return singular if (num == 1 or num == -1) else plural or singular + 's'

freqtrade/pairlist/IPairList.py

@@ -8,6 +8,9 @@
 from abc import ABC, abstractmethod
 from typing import List
 
+from freqtrade.exchange import market_is_active
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -77,7 +80,7 @@ def _validate_whitelist(self, whitelist: List[str]) -> List[str]:
                 continue
             # Check if market is active
             market = markets[pair]
-            if not market['active']:
+            if not market_is_active(market):
                 logger.info(f"Ignoring {pair} from whitelist. Market is not active.")
                 continue
             sanitized_whitelist.add(pair)