Merge branch 'develop' into pr/paranoidandy/8272

.github/workflows/ci.yml

@@ -16,7 +16,8 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-
+permissions:
+  repository-projects: read
 jobs:
   build_linux:
 
@@ -321,7 +322,6 @@ jobs:
   build_linux_online:
     # Run pytest with "live" checks
     runs-on: ubuntu-22.04
-    # permissions:
     steps:
     - uses: actions/checkout@v3
 
@@ -425,15 +425,15 @@ jobs:
         python setup.py sdist bdist_wheel
 
     - name: Publish to PyPI (Test)
-      uses: pypa/gh-action-pypi-publish@v1.6.4
+      uses: pypa/gh-action-pypi-publish@v1.8.1
       if: (github.event_name == 'release')
       with:
         user: __token__
         password: ${{ secrets.pypi_test_password }}
         repository_url: https://test.pypi.org/legacy/
 
     - name: Publish to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.6.4
+      uses: pypa/gh-action-pypi-publish@v1.8.1
       if: (github.event_name == 'release')
       with:
         user: __token__
@@ -466,12 +466,13 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        IMAGE_NAME: freqtradeorg/freqtrade
         BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
       run: |
         build_helpers/publish_docker_multi.sh
 
   deploy_arm:
+    permissions:
+      packages: write
     needs: [ deploy ]
     # Only run on 64bit machines
     runs-on: [self-hosted, linux, ARM64]
@@ -494,8 +495,9 @@ jobs:
 
     - name: Build and test and push docker images
       env:
-        IMAGE_NAME: freqtradeorg/freqtrade
         BRANCH_NAME: ${{ steps.extract_branch.outputs.branch }}
+        GHCR_USERNAME: ${{ github.actor }}
+        GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         build_helpers/publish_docker_arm64.sh
 

.pre-commit-config.yaml

@@ -17,8 +17,8 @@ repos:
           - types-filelock==3.2.7
           - types-requests==2.28.11.15
           - types-tabulate==0.9.0.1
-          - types-python-dateutil==2.8.19.9
-          - SQLAlchemy==2.0.4
+          - types-python-dateutil==2.8.19.10
+          - SQLAlchemy==2.0.7
         # stages: [push]
 
   - repo: https://github.com/pycqa/isort
@@ -30,7 +30,7 @@ repos:
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.251'
+    rev: 'v0.0.255'
     hooks:
       - id: ruff
 

build_helpers/install_ta-lib.sh

@@ -8,8 +8,8 @@ if [ -n "$2" ] || [ ! -f "${INSTALL_LOC}/lib/libta_lib.a" ]; then
   tar zxvf ta-lib-0.4.0-src.tar.gz
   cd ta-lib \
   && sed -i.bak "s|0.00000001|0.000000000000000001 |g" src/ta_func/ta_utility.h \
-  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' -o config.guess \
-  && curl 'http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' -o config.sub \
+  && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.guess' -o config.guess \
+  && curl 'https://raw.githubusercontent.com/gcc-mirror/gcc/master/config.sub' -o config.sub \
   && ./configure --prefix=${INSTALL_LOC}/ \
   && make
   if [ $? -ne 0 ]; then

build_helpers/publish_docker_arm64.sh

@@ -3,6 +3,10 @@
 # Use BuildKit, otherwise building on ARM fails
 export DOCKER_BUILDKIT=1
 
+IMAGE_NAME=freqtradeorg/freqtrade
+CACHE_IMAGE=freqtradeorg/freqtrade_cache
+GHCR_IMAGE_NAME=ghcr.io/freqtrade/freqtrade
+
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 TAG_PLOT=${TAG}_plot
@@ -14,7 +18,6 @@ TAG_ARM=${TAG}_arm
 TAG_PLOT_ARM=${TAG_PLOT}_arm
 TAG_FREQAI_ARM=${TAG_FREQAI}_arm
 TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL}_arm
-CACHE_IMAGE=freqtradeorg/freqtrade_cache
 
 echo "Running for ${TAG}"
 
@@ -38,13 +41,13 @@ if [ $? -ne 0 ]; then
     echo "failed building multiarch images"
     return 1
 fi
-# Tag image for upload and next build step
-docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM
 
 docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot .
 docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai .
 docker build --cache-from freqtrade:${TAG_ARM} --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl .
 
+# Tag image for upload and next build step
+docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM
 docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM
 docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM
 docker tag freqtrade:$TAG_FREQAI_RL_ARM ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM
@@ -59,7 +62,6 @@ fi
 
 docker images
 
-# docker push ${IMAGE_NAME}
 docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM
 docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM
 docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM
@@ -82,14 +84,30 @@ docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI}
 docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM}
 docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_RL}
 
+# copy images to ghcr.io
+
+alias crane="docker run --rm -i -v $(pwd)/.crane:/home/nonroot/.docker/ gcr.io/go-containerregistry/crane"
+mkdir .crane
+chmod a+rwx .crane
+
+echo "${GHCR_TOKEN}" | crane auth login ghcr.io -u "${GHCR_USERNAME}" --password-stdin
+
+crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL}
+crane copy ${IMAGE_NAME}:${TAG_FREQAI} ${GHCR_IMAGE_NAME}:${TAG_FREQAI}
+crane copy ${IMAGE_NAME}:${TAG_PLOT} ${GHCR_IMAGE_NAME}:${TAG_PLOT}
+crane copy ${IMAGE_NAME}:${TAG} ${GHCR_IMAGE_NAME}:${TAG}
+
 # Tag as latest for develop builds
 if [ "${TAG}" = "develop" ]; then
     echo 'Tagging image as latest'
     docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG}
     docker manifest push -p ${IMAGE_NAME}:latest
+
+    crane copy ${IMAGE_NAME}:latest ${GHCR_IMAGE_NAME}:latest
 fi
 
 docker images
+rm -rf .crane
 
 # Cleanup old images from arm64 node.
 docker image prune -a --force --filter "until=24h"

build_helpers/publish_docker_multi.sh

@@ -2,6 +2,8 @@
 
 # The below assumes a correctly setup docker buildx environment
 
+IMAGE_NAME=freqtradeorg/freqtrade
+CACHE_IMAGE=freqtradeorg/freqtrade_cache
 # Replace / with _ to create a valid tag
 TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
 TAG_PLOT=${TAG}_plot
@@ -11,7 +13,6 @@ TAG_PI="${TAG}_pi"
 
 PI_PLATFORM="linux/arm/v7"
 echo "Running for ${TAG}"
-CACHE_IMAGE=freqtradeorg/freqtrade_cache
 CACHE_TAG=${CACHE_IMAGE}:${TAG_PI}_cache
 
 # Add commit and commit_message to docker container

docs/bot-basics.md

@@ -12,6 +12,9 @@ This page provides you some basic concepts on how Freqtrade works and operates.
 * **Indicators**: Technical indicators (SMA, EMA, RSI, ...).
 * **Limit order**: Limit orders which execute at the defined limit price or better.
 * **Market order**: Guaranteed to fill, may move price depending on the order size.
+* **Current Profit**: Currently pending (unrealized) profit for this trade. This is mainly used throughout the bot and UI.
+* **Realized Profit**: Already realized profit. Only relevant in combination with [partial exits](strategy-callbacks.md#adjust-trade-position) - which also explains the calculation logic for this.
+* **Total Profit**: Combined realized and unrealized profit. The relative number (%) is calculated against the total investment in this trade.
 
 ## Fee handling
 

docs/freqai-parameter-table.md

@@ -46,7 +46,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `outlier_protection_percentage` | Enable to prevent outlier detection methods from discarding too much data. If more than `outlier_protection_percentage` % of points are detected as outliers by the SVM or DBSCAN, FreqAI will log a warning message and ignore outlier detection, i.e., the original dataset will be kept intact. If the outlier protection is triggered, no predictions will be made based on the training dataset. <br> **Datatype:** Float. <br> Default: `30`.
 | `reverse_train_test_order` | Split the feature dataset (see below) and use the latest data split for training and test on historical split of the data. This allows the model to be trained up to the most recent data point, while avoiding overfitting. However, you should be careful to understand the unorthodox nature of this parameter before employing it. <br> **Datatype:** Boolean. <br> Default: `False` (no reversal).
 | `shuffle_after_split` | Split the data into train and test sets, and then shuffle both sets individually. <br> **Datatype:** Boolean. <br> Default: `False`.
-| `buffer_train_data_candles` | Cut `buffer_train_data_candles` off the beginning and end of the training data *after* the indicators were populated. The main example use is when predicting maxima and minima, the argrelextrema function  cannot know the maxima/minima at the edges of the timerange. To improve model accuracy, it is best to compute argrelextrema on the full timerange and then use this function to cut off the edges (buffer) by the kernel. In another case, if the targets are set to a shifted price movement, this buffer is unnecessary because the shifted candles at the end of the timerange will be NaN and FreqAI will automatically cut those off of the training dataset.<br> **Datatype:** Boolean. <br> Default: `False`.
+| `buffer_train_data_candles` | Cut `buffer_train_data_candles` off the beginning and end of the training data *after* the indicators were populated. The main example use is when predicting maxima and minima, the argrelextrema function  cannot know the maxima/minima at the edges of the timerange. To improve model accuracy, it is best to compute argrelextrema on the full timerange and then use this function to cut off the edges (buffer) by the kernel. In another case, if the targets are set to a shifted price movement, this buffer is unnecessary because the shifted candles at the end of the timerange will be NaN and FreqAI will automatically cut those off of the training dataset.<br> **Datatype:** Integer. <br> Default: `0`.
 
 ### Data split parameters
 
@@ -84,6 +84,7 @@ Mandatory parameters are marked as **Required** and have to be set in one of the
 | `add_state_info` | Tell FreqAI to include state information in the feature set for training and inferencing. The current state variables include trade duration, current profit, trade position. This is only available in dry/live runs, and is automatically switched to false for backtesting. <br> **Datatype:** bool. <br> Default: `False`.
 | `net_arch` | Network architecture which is well described in [`stable_baselines3` doc](https://stable-baselines3.readthedocs.io/en/master/guide/custom_policy.html#examples). In summary: `[<shared layers>, dict(vf=[<non-shared value network layers>], pi=[<non-shared policy network layers>])]`. By default this is set to `[128, 128]`, which defines 2 shared hidden layers with 128 units each.
 | `randomize_starting_position` | Randomize the starting point of each episode to avoid overfitting. <br> **Datatype:** bool. <br> Default: `False`.
+| `drop_ohlc_from_features` | Do not include the normalized ohlc data in the feature set passed to the agent during training (ohlc will still be used for driving the environment in all cases) <br> **Datatype:** Boolean. <br> **Default:** `False`
 
 ### Additional parameters
 

docs/freqai-reinforcement-learning.md

@@ -176,9 +176,11 @@ As you begin to modify the strategy and the prediction model, you will quickly r
 
                 factor = 100
 
+                pair = self.pair.replace(':', '')
+
                 # you can use feature values from dataframe
                 # Assumes the shifted RSI indicator has been generated in the strategy.
-                rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{self.pair}_"
+                rsi_now = self.raw_features[f"%-rsi-period-10_shift-1_{pair}_"
                                 f"{self.config['timeframe']}"].iloc[self._current_tick]
 
                 # reward agent for entering trades
@@ -246,13 +248,13 @@ FreqAI also provides a built in episodic summary logger called `self.tensorboard
             """
             def calculate_reward(self, action: int) -> float:
                 if not self._is_valid(action):
-                    self.tensorboard_log("is_valid")
+                    self.tensorboard_log("invalid")
                     return -2
 
 ```
 
 !!! Note
-    The `self.tensorboard_log()` function is designed for tracking incremented objects only i.e. events, actions inside the training environment. If the event of interest is a float, the float can be passed as the second argument e.g. `self.tensorboard_log("float_metric1", 0.23)` would add 0.23 to `float_metric`. In this case you can also disable incrementing using `inc=False` parameter.
+    The `self.tensorboard_log()` function is designed for tracking incremented objects only i.e. events, actions inside the training environment. If the event of interest is a float, the float can be passed as the second argument e.g. `self.tensorboard_log("float_metric1", 0.23)`. In this case the metric values are not incremented.
 
 ### Choosing a base environment
 

docs/freqai-running.md

@@ -128,6 +128,9 @@ The FreqAI specific parameter `label_period_candles` defines the offset (number
 
 You can choose to adopt a continual learning scheme by setting `"continual_learning": true` in the config. By enabling `continual_learning`, after training an initial model from scratch, subsequent trainings will start from the final model state of the preceding training. This gives the new model a "memory" of the previous state. By default, this is set to `False` which means that all new models are trained from scratch, without input from previous models.
 
+???+ danger "Continual learning enforces a constant parameter space"
+    Since `continual_learning` means that the model parameter space *cannot* change between trainings, `principal_component_analysis` is automatically disabled when `continual_learning` is enabled. Hint: PCA changes the parameter space and the number of features, learn more about PCA [here](freqai-feature-engineering.md#data-dimensionality-reduction-with-principal-component-analysis).
+
 ## Hyperopt
 
 You can hyperopt using the same command as for [typical Freqtrade hyperopt](hyperopt.md):

docs/includes/protections.md

@@ -149,7 +149,7 @@ The below example assumes a timeframe of 1 hour:
 * Locks each pair after selling for an additional 5 candles (`CooldownPeriod`), giving other pairs a chance to get filled.
 * Stops trading for 4 hours (`4 * 1h candles`) if the last 2 days (`48 * 1h candles`) had 20 trades, which caused a max-drawdown of more than 20%. (`MaxDrawdown`).
 * Stops trading if more than 4 stoploss occur for all pairs within a 1 day (`24 * 1h candles`) limit (`StoplossGuard`).
-* Locks all pairs that had 4 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`).
+* Locks all pairs that had 2 Trades within the last 6 hours (`6 * 1h candles`) with a combined profit ratio of below 0.02 (<2%) (`LowProfitPairs`).
 * Locks all pairs for 2 candles that had a profit of below 0.01 (<1%) within the last 24h (`24 * 1h candles`), a minimum of 4 trades.
 
 ``` python

docs/requirements-docs.txt

@@ -1,6 +1,6 @@
 markdown==3.3.7
 mkdocs==1.4.2
-mkdocs-material==9.0.15
+mkdocs-material==9.1.3
 mdx_truly_sane_lists==1.3
-pymdown-extensions==9.9.2
+pymdown-extensions==9.10
 jinja2==3.1.2

docs/strategy-callbacks.md

@@ -318,11 +318,11 @@ class AwesomeStrategy(IStrategy):
 
         # evaluate highest to lowest, so that highest possible stop is used
         if current_profit > 0.40:
-            return stoploss_from_open(0.25, current_profit, is_short=trade.is_short)
+            return stoploss_from_open(0.25, current_profit, is_short=trade.is_short, leverage=trade.leverage)
         elif current_profit > 0.25:
-            return stoploss_from_open(0.15, current_profit, is_short=trade.is_short)
+            return stoploss_from_open(0.15, current_profit, is_short=trade.is_short, leverage=trade.leverage)
         elif current_profit > 0.20:
-            return stoploss_from_open(0.07, current_profit, is_short=trade.is_short)
+            return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage)
 
         # return maximum stoploss value, keeping current stoploss price unchanged
         return 1

docs/strategy-customization.md

@@ -881,14 +881,16 @@ All columns of the informative dataframe will be available on the returning data
 
 ### *stoploss_from_open()*
 
-Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the open price instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired percentage above the open price.
+Stoploss values returned from `custom_stoploss` must specify a percentage relative to `current_rate`, but sometimes you may want to specify a stoploss relative to the entry point instead. `stoploss_from_open()` is a helper function to calculate a stoploss value that can be returned from `custom_stoploss` which will be equivalent to the desired trade profit above the entry point.
 
 ??? Example "Returning a stoploss relative to the open price from the custom stoploss function"
 
     Say the open price was $100, and `current_price` is $121 (`current_profit` will be `0.21`).  
 
     If we want a stop price at 7% above the open price we can call `stoploss_from_open(0.07, current_profit, False)` which will return `0.1157024793`.  11.57% below $121 is $107, which is the same as 7% above $100.
 
+    This function will consider leverage - so at 10x leverage, the actual stoploss would be 0.7% above $100 (0.7% * 10x = 7%).
+
 
     ``` python
 
@@ -907,7 +909,7 @@ Stoploss values returned from `custom_stoploss` must specify a percentage relati
 
             # once the profit has risen above 10%, keep the stoploss at 7% above the open price
             if current_profit > 0.10:
-                return stoploss_from_open(0.07, current_profit, is_short=trade.is_short)
+                return stoploss_from_open(0.07, current_profit, is_short=trade.is_short, leverage=trade.leverage)
 
             return 1
 
@@ -1038,11 +1040,10 @@ from datetime import timedelta, datetime, timezone
 
 # Within populate indicators (or populate_buy):
 if self.config['runmode'].value in ('live', 'dry_run'):
-   # fetch closed trades for the last 2 days
-    trades = Trade.get_trades([Trade.pair == metadata['pair'],
-                               Trade.open_date > datetime.utcnow() - timedelta(days=2),
-                               Trade.is_open.is_(False),
-                ]).all()
+    # fetch closed trades for the last 2 days
+    trades = Trade.get_trades_proxy(
+        pair=metadata['pair'], is_open=False, 
+        open_date=datetime.now(timezone.utc) - timedelta(days=2))
     # Analyze the conditions you'd like to lock the pair .... will probably be different for every strategy
     sumprofit = sum(trade.close_profit for trade in trades)
     if sumprofit < 0: