From e15f9bcc335a21ec20a59e3c2bd151843d6be957 Mon Sep 17 00:00:00 2001 From: wkirgsn Date: Wed, 21 Oct 2020 11:20:25 +0200 Subject: [PATCH] polish up gem tutorial. Polish up README (links to colab missing yet). Other tutorials also need polishment yet --- README.md | 299 +---- docs/plots/SCML_Overview.svg | 1137 +++++++---------- examples/example_notebooks/GEM_cookbook.ipynb | 740 +++++++++++ examples/example_notebooks/GEM_tutorial.ipynb | 515 -------- 4 files changed, 1288 insertions(+), 1403 deletions(-) create mode 100644 examples/example_notebooks/GEM_cookbook.ipynb delete mode 100644 examples/example_notebooks/GEM_tutorial.ipynb diff --git a/README.md b/README.md index 96307d29..3d28cbf6 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,45 @@ +# Gym Electric Motor ![](docs/plots/Motor_Logo.png) +[**Paper**](https://arxiv.org/abs/1910.09434) +| [**Quickstart**](#getting-started) +| [**Install guide**](#installation) +| [**Reference docs**](https://upb-lea.github.io/gym-electric-motor/) +| [**Release notes**](https://github.com/upb-lea/gym-electric-motor/releases) + [![Build Status](https://travis-ci.org/upb-lea/gym-electric-motor.svg?branch=master)](https://travis-ci.org/upb-lea/gym-electric-motor) [![codecov](https://codecov.io/gh/upb-lea/gym-electric-motor/branch/master/graph/badge.svg)](https://codecov.io/gh/upb-lea/gym-electric-motor) [![PyPI version shields.io](https://img.shields.io/pypi/v/gym-electric-motor.svg)](https://pypi.python.org/pypi/gym-electric-motor/) [![License](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/upb-lea/gym-electric-motor/blob/master/LICENSE) -__The gym-electric-motor (GEM) package is a software toolbox for the -simulation of -different electric motors.__ +## Overview +The gym-electric-motor (GEM) package is a Python toolbox for the simulation and control of various electric motors. -The toolbox is built upon the [OpenAI Gym Environments](https://gym.openai.com/) for reinforcement learning. -Therefore, the toolbox is specifically designed for running reinforcement -learning algorithms to train agents controlling electric motors. +It is built upon [OpenAI Gym Environments](https://gym.openai.com/), and, therefore, can be used for both, classical control simulation and reinforcement learning experiments. It allows you to construct a typical drive train with the usual building blocks, i.e. supply voltages, converters, electric motors and load models, and obtain not only a closed-loop simulation of this physical structure, but also a rich interface for plugging in any decision making algorithm, from PI-controllers to [Deep Deterministic Policy Gradient](https://spinningup.openai.com/en/latest/algorithms/ddpg.html) agents. -[Read the detailed docs!](https://upb-lea.github.io/gym-electric-motor/) +## Getting Started +An easy way to get started with GEM is by playing around with the following interactive notebooks in Google Colaboratory. Most important features of GEM as well as application demonstrations are showcased, and give a kickstart for engineers in industry and academia. -So far, several DC-motor models and the three-phase motors permanent magnet synchronous motor (PMSM), -synchronous reluctance motor (SynRM), squirrel cage induction motor (SCIM) and doubly-fed induction -motor (DFIM) are available. -Beside electrical motors, also converters and load models are implemented. The converters can be driven by means of a duty cycle (continuous mode) or -switching commands (discrete mode). -The figure shows the basic scheme of the converter, motor and load. -### Physical Structure of the Environment's Components -![](docs/plots/SCML_Setting.svg) -### Control Flow of a Step Cycle of the Environment -![](docs/plots/CycleScheme.svg) -### Citation -A whitepaper for this framework is available under [arxiv.org/abs/1910.09434 -](https://arxiv.org/abs/1910.09434). Please use the following BibTeX entry for citing us: -``` -@misc{traue2019reinforcement, - title={Towards a Reinforcement Learning Environment Toolbox for Intelligent Electric Motor Control}, - author={Arne Traue and Gerrit Book and Wilhelm Kirchgässner and Oliver Wallscheid}, - year={2019}, - eprint={1910.09434}, - archivePrefix={arXiv}, - primaryClass={eess.SY} -} +There is a list of [standalone example scripts](examples/) as well for minimalistic demonstrations. + +A basic routine is as simple as: +```py +import gym_electric_motor as gem + +if __name__ == '__main__': + env = gem.make("PMSMDisc-v1") # instantiate a discretely controlled PMSM + env.reset() + for _ in range(1000): + env.render() # visualize environment + (states, references), rewards, done, _ =\ + env.step(env.action_space.sample()) # pick random control actions + env.close() ``` -### Installing + +## Installation - Install gym-electric-motor from PyPI (recommended): ``` @@ -58,224 +56,39 @@ python setup.py install # or alternatively pip install -e . ``` -### Authors -* Praneeth Balakrishna -* Gerrit Book -* Wilhelm Kirchgässner -* Maximilian Schenke -* Arne Traue -* Oliver Wallscheid - -### Getting started -Like every gym environment, the basic user interface consists of four main functions. -* `import gym_electric_motor as gem` - Import of the package. -* `env = gem.make(environment-id, **kwargs)` - Returns an instantiated motor environment. Call this function at the beginning. - The `gem.make()` method is equal to the `gym.make()`. By using `gem.make()`you can avoid importing gym additionally. +## Building Blocks +A GEM environment consists of following building blocks: +- Physical Structure: + - Supply Voltage + - Converter + - Electric motor + - Load Model +- Utility functions for reference generation, reward calculation and visualization -* `(initial_state, initial_reference) = env.reset()` - Resets the motor. This includes a new initial state and new reference trajectories. - Call this function before a new episode starts. - -* `(state, reference), reward, done, _ = env.step(action)` - Simulate one single time step on the motor with an action. - Call this function iteratively until termination is reached. - -* `env.render()` - Update the visualization of the motor states. +### Information Flow in a GEM Environment +![](docs/plots/SCML_Overview.svg) -### GEM Make Call -The make function takes the environment-ids and several constructor arguments. -Every environment also works without further parameters with default values. -These default parameters can be looked up in the API-documentation of every GEM-environment. -With the environment-id you select a certain motor type and action type (continuous or discrete) and with the further -constructor arguments you can parametrize the environment to your control problem. - -##### Environment Ids - -* DC Motors - - * `'DcExtExCont-v1'` - Externally excited DC motor with continuous actions. - - * `'DcExtExDisc-v1'` - Externally excited DC motor with discrete actions. - - * `'DcPermExCont-v1'` - Permanently excited DC motor with continuous actions. - - * `'DcPermexDisc-v1'` - Permanently excited DC motor with discrete actions. - - * `'DcShuntCont-v1'` - DC shunt motor with continuous actions. - - * `'DcShuntDisc-v1'` - DC shunt motor with discrete actions. - - * `'DcSeriesCont-v1'` - DC series motor with continuous actions. - - * `'DcSeriesDisc-v1'` - DC series motor with discrete actions. - -* Synchronous Motors - - * `'PMSMCont-v1'`: - Permanent magnet synchronous motor with continuous actions. - - * `'PMSMDisc-v1'`: - Permanent magnet synchronous motor with discrete actions. - - * `'SynRMCont-v1'`: - Synchronous reluctance motor with continuous actions. - - * `'SynRMDisc-v1'`: - Synchronous reluctance motor with discrete actions. - -* Induction Motors - - * `'SCIMCont-v1'`: - Squirrel cage induction motor with continuous actions. - - * `'SCIMDisc-v1'`: - Squirrel cage induction motor with discrete actions. - - * `'DFIMCont-v1'`: - Doubly fed induction motor with continuous actions. - - * `'DFIMDisc-v1'`: - Doubly fed induction motor with discrete actions. - -#### Make Keyword-Arguments -Using the keyword arguments in the `gem.make(id, **kwargs)` function you can select different function modules for the -environment and parametrize these modules. -The main level modules of each GEM-environment consists of four function modules: - -* Physical System - * keyword: `physical_system` - * Specification and simulation of the system model. -* Reference Generator - * keyword: `reference_generator` - * Generation of references that the physical system has to follow. -* Reward Function - * keyword: `reward_function` - * Reward calculation based on the current state and reference. -* Visualization - * keyword: `visualization` - * Visualization of the physical systems state, reference and rewards. -* State Filter - * keyword: `state_filter` - * Selection of states that shall be shown to the agent. - -These function modules can be selected in three ways: - -* Passing a keystring (and further keyword arguments for the class) : - * `reference_generator='SinusoidalReference', amplitude_range=(0.2, 0.8)` - -* Passing a class pointer (and further keyword arguments for the class) - * `reference_generator=SinusoidalReferenceGenerator, amplitude_range=(0.2,0.8)` - -* Passing an instantiated object - * `reference_generator = SinusoidalReferenceGenerator(amplitude_range=(0.2,0.8)` - -Furthermore, the internal function modules of the physical systems like the converter, motor, load can be selected in -the make keyword-arguments in the same way. - -The available modules and specific keyword-arguments for each module can be looked up in the API-documentation. -### Reset -The reset function determines new references, new initial values and resets the visualization. -Call this function before a new episode begins. -The parameters of the motor, converter and load will be those during instantiation. - -### Step -This function performs one action on the environment for one time step. -It simulates the motor and needs to be called in every time step. It takes the action as parameter only. -First the input voltage to the motor from the converter is determined and afterwards an integrator is used to compute -the next state. -Eventually, the reward is evaluated and returned together with the next observation and a flag indicating termination. -Several reward functions are available. - -### Render -The visualization contains graphs of the motor quantities 'speed, voltages, currents, torque' for one episode. -What should be shown is to be specified in the configuration-parameter. -The quantities that should be displayed can be specified in the constructor-parameters. -All visualizations are optional and recommended to be disabled for increased speed of training. - -### Examples - -- Conventional PI controller as speed controller on a dc series motor [(jump to source)](examples/pi_series_omega_control.py). - -- Training and testing of a [Keras-rl](https://github.com/keras-rl/keras-rl) DDPG-Agent as a speed controller on a dc series motor [(jump to source)](examples/ddpg_series_omega_control.py). - -### Physical System Models -The following electrical motor , converter and mechanical load models are included. More detailed descriptions can be found in the corresponding classes. - -##### Motor Models -The following motor models are included: +Among various DC-motor models, the following three-phase motors - together with their power electronic counterparts - are available: +- Permanent Magnet Synchronous Motor (PMSM), +- Synchronous Reluctance Motor (SynRM) +- Squirrel Cage Induction Motor (SCIM) +- Doubly-fed Induction Motor (DFIM) -Four DC motors: - -- permanently excited motor -- externally excited motor -- series motor -- shunt motor - -Two three phase motors: - -- PMSM (permanent magnet synchronous motor) -- SynRM (synchronous reluctance motor) - -Two variants of the induction motor: - -- SCIM (squirrel cage induction motor) -- DFIM (doubly fed induction motor) - -##### Converter -Following converters are included: - -- 1 quadrant converter (1QC) - -- 2 quadrant converter (2QC) as an asymmetric half bridge with both current polarities - -- 4 quadrant converter (4QC) - -- B6 Bridge Converter (B6C) - -All converters can consider interlocking times and a dead time of one sampling interval. -Furthermore, they can be controlled with a discrete action space or a continuous action space. - -Discrete actions are the direct switching states of the transistors. -Continuous actions are the duty cycles for a pulse width modulation on the transistors. - -##### Load -The load model consists of a quadratic load function, with user defined coefficients. -Furthermore the moment of inertia of the load attached to the motor can be specified. - -### Notes about the states and value ranges -The included states for each motor are summarized and briefly described in the [Motor Dashboard](visualizations/motor_dashboard.html). -Every state that can be plotted can also be used in the state filter or as observed state. -The actions are basically understood as the desired duty cycles. The actual applied voltage can be taken from the observations. -The observations are normalized to their physical limits that can be accessed with `env.limits`. -Therefore, all values are typically in a range of [0, 1] or [-1, 1] without limit violation. - - - -### Notes about the Parameters -All nominal values of voltages and currents are DC values in the case of a DC motor and peak phase values for the PMSM. -Therefore, data sheet values for line voltage and phase currents of a PMSM has to be transformed with: - -![](docs/plots/voltagetransformation.svg) - -Furthermore, the angular velocity is the mechanical one and not the electrical: - -![](docs/plots/omegame.svg) - -The mechanical one is needed for speed controller. -The included angle is the electrical one due to its usage in the rotational transformation for three phase motors. +The converters can be driven by means of a duty cycle (continuous mode) or switching commands (discrete mode). +### Citation +A whitepaper for this framework is available under [arxiv.org/abs/1910.09434](https://arxiv.org/abs/1910.09434). Please use the following BibTeX entry for citing us: +``` +@misc{traue2019reinforcement, + title={Towards a Reinforcement Learning Environment Toolbox for Intelligent Electric Motor Control}, + author={Arne Traue and Gerrit Book and Wilhelm Kirchgässner and Oliver Wallscheid}, + year={2019}, + eprint={1910.09434}, + archivePrefix={arXiv}, + primaryClass={eess.SY} +} +``` ### Running Unit Tests with Pytest To run the unit tests ''pytest'' is required. diff --git a/docs/plots/SCML_Overview.svg b/docs/plots/SCML_Overview.svg index e37b209a..75a4cae6 100644 --- a/docs/plots/SCML_Overview.svg +++ b/docs/plots/SCML_Overview.svg @@ -642,13 +642,13 @@ inkscape:collect="always" style="color-interpolation-filters:sRGB" id="filter28043" - x="-1.6697379e-05" - width="1.0000334" - y="-2.2585074e-05" - height="1.0000452"> + x="-1.7761754e-05" + width="1.0000355" + y="-2.0891691e-05" + height="1.0000418"> @@ -659,11 +659,11 @@ borderopacity="1.0" inkscape:pageopacity="0.0" inkscape:pageshadow="2" - inkscape:zoom="0.67968205" - inkscape:cx="545.32308" - inkscape:cy="452.82911" + inkscape:zoom="0.48060778" + inkscape:cx="-80.821359" + inkscape:cy="711.63732" inkscape:document-units="px" - inkscape:current-layer="layer3" + inkscape:current-layer="layer2" showgrid="true" inkscape:window-width="1848" inkscape:window-height="1016" @@ -690,7 +690,7 @@ image/svg+xml - + @@ -714,7 +714,7 @@ inkscape:groupmode="layer" id="layer3" inkscape:label="BG" - style="display:inline;opacity:1;mix-blend-mode:normal;filter:url(#filter28043)" + style="display:inline;opacity:1;mix-blend-mode:normal;" transform="translate(196.21502,502.95572)"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -GEM - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + style="display:inline"> + + + + + + + GEM + + id="g12" + transform="matrix(1.1622476,0,0,1.1622476,709.67363,-234.77045)"> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Environment " + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import gym_electric_motor as gem\n", + "\n", + "basic_env = gem.make(\"PMSMDisc-v1\") # pass a motor environment ID \n", + "basic_env" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Physical System\n", + "Each system consists of a voltage supply, a power electronic converter, an electrical motor, and the mechanical load (SCML) as shown in the above figure. Each such SCML-system is simulated by a user-defined ODE solver.\n", + "\n", + "#### Voltage Supply\n", + "\n", + "The voltage supply module provides both, DC and AC voltage supplies. \n", + "- The DC supplies are either ideal or non-ideal voltage sources.\n", + "- The AC supplies are either ideal single phase or ideal three-phase AC sources.\n", + "\n", + "More documentation regarding the voltage supplies of GEM can be found [here](https://upb-lea.github.io/gym-electric-motor/parts/physical_systems/voltage_supplies/voltage_supply.html). \n", + "For the PMSM environment example, a non-ideal DC voltage supply is created. \n", + "Here, the DC-link is modeled as an RC-circuit loaded from an ideal DC voltage source. \n", + "\n", + "The non-ideal DC supply in GEM is named 'RCVoltageSupply' and the supply_parameter(dict) consists of resistance R in ohm and capacitance C in farad\n" + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "supply = 'RCVoltageSupply'\n", + "supply_parameter=dict(R=10, C=4e-3) # Note, R and C values here are not realistic." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Converter \n", + "The converters are divided into two classes: The continuously controlled and the discretely controlled converters.\n", + "\n", + "In the continuous case, the agent's actions denote a duty cycle that is modulated to the converter's output voltage.\n", + "\n", + "In the discrete case, the agent's actions denote switching states of the converter at the given instant.\n", + "Here, only a discrete amount of options are available. In this notebook, for the PMSM the discrete B6 bridge converter with six switches is utilized per default. This converter provides a total of eight possible actions.\n", + "![](../../docs/plots/B6.svg)\n", + "\n", + "\n", + "#### Motor \n", + "In this tutorial, the electric motor used is the **Permanent Magnet Synchronous Motor**.\n", + "The motor schematic is the following:\n", + "\n", + "\n", + "![](../../docs/plots/ESBdq.svg)\n", + "\n", + "And the electrical ODEs for that motor are:\n", + "\n", + "

\n", + "\n", + "\n", + "\n", + " $ \\frac{\\mathrm{d}i_{sd}}{\\mathrm{d}t}=\\frac{u_{sd} + p\\omega_{me}L_q i_{sq} - R_s i_{sd}}{L_d} $

\n", + " $\\frac{\\mathrm{d} i_{sq}}{\\mathrm{d} t}=\\frac{u_{sq} - p \\omega_{me} (L_d i_{sd} + \\mathit{\\Psi}_p) - R_s i_{sq}}{L_q}$

\n", + " $\\frac{\\mathrm{d}\\epsilon_{el}}{\\mathrm{d}t} = p\\omega_{me}$\n", + "\n", + "

\n", + " \n", + "\n", + "\n", + "\n", + "The motor environment ID for the discrete PMSM motor is **\"PMSMDisc-v1\"**.\n", + "The parameters of the specific motor are to be passed by the user as a motor parameter dictionary.\n", + "Default parameters will be considered in case the motor parameters are not provided by the user. \n", + "The nominal and limit values which define the operating region of the motor are also passed as a dictionary." + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "motor_env_id = \"PMSMDisc-v1\"\n", + "tau = 1e-5 # The duration of each sampling step\n", + "motor_parameter = dict(p=3, # [p] = 1, nb of pole pairs\n", + " r_s=17.932e-3, # [r_s] = Ohm, stator resistance\n", + " l_d=0.37e-3, # [l_d] = H, d-axis inductance\n", + " l_q=1.2e-3, # [l_q] = H, q-axis inductance\n", + " psi_p=65.65e-3, # [psi_p] = Vs, magnetic flux of the permanent magnet\n", + " ) # BRUSA\n", + "\n", + "nominal_values=dict(omega=4000*2*np.pi/60, # angular velocity in rpm\n", + " i=230, # motor current in amps\n", + " u=350 # nominal voltage in volts\n", + " )\n", + "# limit values are taken exemplarily as 1.3 times the nominal values\n", + "limit_values = {key: 1.5 * nomin for key, nomin in nominal_values.items()}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Motor state initializer\n", + "By default, the motor states (e.g. motor currents, rotational speed) are always set to zero whenever the motor environment is reset. In order to generate diverse expisodes, the motor state initializer can be used to draw the initial state values from a given probability distribution within the nominal operating range of the given motor. \n", + "\n", + "The 'motor_initializer' is a dictionary that consists of the type of distribution, for example, uniform or gaussian distribution and the interval of values within the nominal operating region. \n", + "Here, the motor states, i.e. $i_{sd}$, $i_{sq}$ and motor angle are initialized with values sampled from a uniform distribution from the provided intervals.\n", + "\n", + "$i_{sd}$ and $i_{sq}$ are the motor currents in the [d-q coordinate system](https://en.wikipedia.org/wiki/Direct-quadrature-zero_transformation#:~:text=The%20direct%2Dquadrature%2Dzero%20,an%20effort%20to%20simplify%20analysis).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "motor_initializer={'random_init': 'uniform', 'interval': [[-230, 230], [-230, 230], [-np.pi, np.pi]]} " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Mechanical Load\n", + "The mechanical load module in GEM models various external mechanical systems that apply counterforces on the electrical motor's rotor through the drive shaft.\n", + "Different types of load models such as constant speed load and polynomial static load are provided. More information can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/physical_systems/mechanical_loads/mechanical_load.html)\n", + "\n", + "In this example, the load type: _ConstSpeedLoad_ is used. \n", + "This initializes the load with a constant speed at the start of each episode. \n", + "The initialization value for speed is sampled from a uniform distribution defined by the given interval." + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [], + "source": [ + "from gym_electric_motor.physical_systems import ConstantSpeedLoad\n", + "\n", + "load = 'ConstSpeedLoad'\n", + "load_initializer={'random_init': 'uniform', 'interval':[100,200] } \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 Reward Function\n", + "The reward calculation is based on the current state and reference of the motor environment. It is calculated as a weighted sum of errors with a certain power as follows:\n", + "\n", + "\n", + "\n", + "\n", + "$ r_t = - \\sum \\limits_{k=0} ^{N} w_{\\{k\\}}\\big | s_{\\{k\\}t} - s^*_{\\{k\\}t} \\big |^p $

\n", + "\n", + "Here, $r_t$ is the reward at time $t$. $ w_{\\{k\\}}$ is the reward weight for the k'th state variable of the $N$ referenced states . $s_{\\{k\\}t}$ is the k'th state variable of the motor and $s^*_{\\{k\\}t}$ is the corresponding k'th reference variable at time $t$. The reward power is $p$.\n", + "\n", + "\n", + "If states are to be monitored for a constraint violation, an additional terminal reward is added. This value depends on the discount factor $\\gamma$ as follows. \n", + "\n", + "$r_t = -1 / (1 - \\gamma).$\n", + "\n", + "\n", + "### 2.4 Constraint Monitor\n", + "The constraint monitor monitors the system states and assesses whether they comply with the given limits or violate them. \n", + "It provides any necessary information to the reward function.\n", + "The constraints of the system states can be generally described by the user, and by default is restricted by the physical enviroment limits.\n", + "\n", + "The user defined constraint: \"SqdCurrentMonitor\" presented below, monitors the currents and raises a flag indicating a limit violation if :\n", + "\n", + "$ i_{sd}^2 + i_{sq}^2 > i_{max}^2 $ \n", + "Here $i_{sd}$ and $i_{sq}$ are the motor currents in the d-q coordinate system that can be accessed from the motor states and $i_{max}$ is the maximum allowable current value.\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "gamma = 0.99 #Discount factor for the reward punishment. Should equal agents' discount factor gamma.\n", + "\n", + "def monitor_sqd_currents(state, observed_states, k, physical_system):\n", + " \"\"\"\n", + " monitor for squared currents:\n", + "\n", + " i_sd**2 + i_sq**2 < nominal_limit\n", + " \"\"\"\n", + " I_SD_IDX = physical_system.state_names.index('i_sd') # access motor state i_sd\n", + " I_SQ_IDX = physical_system.state_names.index('i_sq') # access motor state i_sq\n", + " sqd_currents = state[I_SD_IDX] ** 2 + state[I_SQ_IDX] ** 2\n", + " return sqd_currents > 1 " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.5 Reference Generator\n", + "The reference generator produces reference trajectories for the observed states, that the physical system is expected to follow. \n", + "The GEM toolbox provides various reference generators, which can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/reference_generators/reference_generator.html)\n", + "\n", + "In this example, references to the motor currents $i_{sq}$ and $i_{sd}$ are considered.\n", + "The \"WienerProcessReferenceGenerator\" is used to generate random references for both $i_{sq}$ and $i_{sd}$. \n", + "\n", + "The [Wiener process](https://en.wikipedia.org/wiki/Wiener_process) is a stochastic process $W(t)$ for $t>=0$ with $W(0)=0$ such that the increment $W(t)-W(s)$ is Gaussian with mean $0$ and variance $\\sigma$ for any $0<=s" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. A Brief Introduction to GEM\n", - "\n", - "The gym-electric-motor (GEM) package is a software toolbox for the simulation of different electric motors.\n", - "The toolbox is built upon the OpenAI gym environments for reinforcement learning. Therefore, the toolbox is specifically designed for running reinforcement learning algorithms to train agents controlling electric motors.\n", - "Besides electrical motors, converters and load models are also implemented.\n", - "\n", - "The components of GEM are structured as shown in the figure below:\n", - "\n", - "![Motor Setup](img/SCML_Setting.svg)\n", - "\n", - "\n", - "### 1.1 Installation\n", - "Before you can start, you need to make sure that you have gym-electric-motor installed. You can install it easily using pip:\n", - "\n", - "\n", - "pip install gym-electric-motor\n", - "\n", - " \n", - "Alternatively, you can install them and their latest developer version directly from GitHub:\n", - "https://github.com/upb-lea/gym-electric-motor\n", - "\n", - "\n", - "### 1.2 Basic PMSM environment\n", - "As a quick start guide, the following example provides a method to create a basic motor environment . \n", - "By simply calling the gem.make() function with the motor environment ID of the required electric motor, one can create a basic motor environment with default parameters and settings for all the relevant sub-components.\n", - "More information on the motor environment ID and the default parameters can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/environments/environment.html)\n", - "\n", - "However, further sections provide a detailed guide to customize individual sub-components of the motor environment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import gym_electric_motor as gem\n", - "\n", - "basic_env = gem.make(\"PMSMDisc-v1\") # pass the motor environment ID as an arguement to the gem.make()\n", - "basic_env" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Physical System\n", - "The system consists of a voltage supply, a power electronic converter, an electrical motor and the mechanical load (SCML) as shown in the above figure . Additionally, each SCML-system has got an ODE-solver for the simulation.\n", - "\n", - "\n", - "### 2.1 Voltage Supply\n", - "\n", - "The voltage supply module of the GEM toolbox provides both DC and AC voltage supplies. \n", - "- The DC supplies provided are ideal and non-Ideal DC voltage sources.\n", - "- The AC supplies provided are single phase and 3-phase AC sources.
\n", - "\n", - "More documentation regarding the voltage supplies of GEM can be found [here.](https://upb-lea.github.io/gym-electric-motor/parts/physical_systems/voltage_supplies/voltage_supply.html) \n", - "For the PMSM environment example, a non-ideal DC voltage supply is created. \n", - "Here, the DC-link is modeled as an RC-circuit loaded from an ideal DC voltage source. This is illustrated in the below figure. \n", - "\n", - "\n", - "\"non-ideal-supply\"\n", - "\n", - "The non-ideal DC supply in GEM is named 'RCVoltageSupply' and the supply_parameter(dict) consists of resistance R in ohm and capacitance C in farad\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "supply = 'RCVoltageSupply'\n", - "supply_parameter=dict(R=10, C=4e-3) # R and C values here are not realistic. \n", - " # Values here are just for reference" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2 Converter \n", - "The converters are divided into two classes: The continuously controlled and the discretely controlled converters.\n", - "\n", - "In the continuous case, the converter's output voltage is modulated directly through means of duty cycling. In the discrete case, the converter's output voltage is determined by the state of the converter switches at a given instant. Therefore, only a discrete amount of options are available. For this environment, the discrete B6 bridge converter which has three switches which amounts to a total of eight possible actions is used.\n", - "\n", - "\"non-ideal-supply\"\n", - "
\n", - "
\n", - "\n", - "\n", - "### 2.3 Motor \n", - "The electric motor is the **Permanent Magnet Synchronous Motor**.\n", - "The motor schematic is the following:\n", - "\n", - "\n", - "\n", - "\"non-ideal-supply\"\n", - "\n", - "
\n", - "
\n", - "\n", - "And the electrical ODEs for that motor are:\n", - "\n", - "

\n", - "\n", - "\n", - "\n", - " $ \\frac{\\mathrm{d}i_{sd}}{\\mathrm{d}t}=\\frac{u_{sd} + p\\omega_{me}L_q i_{sq} - R_s i_{sd}}{L_d} $

\n", - " $\\frac{\\mathrm{d} i_{sq}}{\\mathrm{d} t}=\\frac{u_{sq} - p \\omega_{me} (L_d i_{sd} + \\mathit{\\Psi}_p) - R_s i_{sq}}{L_q}$

\n", - " $\\frac{\\mathrm{d}\\epsilon_{el}}{\\mathrm{d}t} = p\\omega_{me}$\n", - "\n", - "

\n", - " \n", - "\n", - "\n", - "\n", - "The motor environment ID for the discrete PMSM motor is **\"PMSMDisc-v1\"**. This is later passed to the 'make' function to create the motor environment. The environment IDs for other available motor environments can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/environments/environment.html)\n", - "The parameters of the specific motor is to be passed by the user as a motor parameter dictionary. Default parameters will be considered in case the motor parameters are not provided by the user. The default converter for the environment ID used here is the discrete B6 bridge converter. More details can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/physical_systems/converters/B6C.html)
\n", - "The nominal and limit values which define the operating region of the motor is also passed as a dictionary." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "motor_env_id = \"PMSMDisc-v1\"\n", - "tau = 1e-5 # The duration of each sampling step\n", - "motor_parameter = dict(p=3, # [p] = 1, nb of pole pairs\n", - " r_s=17.932e-3, # [r_s] = Ohm, stator resistance\n", - " l_d=0.37e-3, # [l_d] = H, d-axis inductance\n", - " l_q=1.2e-3, # [l_q] = H, q-axis inductance\n", - " psi_p=65.65e-3, # [psi_p] = Vs, magnetic flux of the permanent magnet\n", - " ) # BRUSA\n", - "\n", - "nominal_values=dict(omega=4000*2*np.pi/60, # angular velocity in RPM\n", - " i=230, # motor current in amps\n", - " u=350 # nominal voltage in volts\n", - " )\n", - "# limit values are taken as 1.3 times the nominal values in this case.\n", - "limit_values = {key: 1.5 * nomin for key, nomin in nominal_values.items()}\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4 Motor state initializer\n", - "By default, the motor states (e.g. motor currents, rotational speed) are always set to zero whenever the motor environment is reset. In order to generate diverse expisodes, the motor state initializer can be used to draw the initial state values from a given probability distribution within the nominal operating range of the given motor. \n", - "\n", - "The 'motor_initializer' is a dictionary that consists of the type of distribution, for example, uniform or gaussian distribution and the interval of values within the nominal operating region. \n", - "Here, the motor states, i.e. $i_{sd}$, $i_{sq}$ and motor angle are initialized with values sampled from a uniform distribution from the provided intervals. $i_{sd}$ and $i_{sq}$ are the motor currents in the d-q coordinate system. More details on the d-q coordinate system can be found [here.](https://en.wikipedia.org/wiki/Direct-quadrature-zero_transformation#:~:text=The%20direct%2Dquadrature%2Dzero%20,an%20effort%20to%20simplify%20analysis)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "motor_initializer={'random_init': 'uniform', 'interval': [[-230, 230], [-230, 230], [-np.pi, np.pi]]} \n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.5 Mechanical Load\n", - "The mechanical load module in GEM models various external mechanical systems attached to the electrical motor's rotor. Different types of load models such as constant speed load and polynomial static load are provided. More information can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/physical_systems/mechanical_loads/mechanical_load.html)\n", - "\n", - "In this example, the load type: \"ConstSpeedLoad\" is used. This initializes the load with a constant speed at the start of each episode. The initialization value for speed is sampled from a uniform distribution defined by the given interval.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from gym_electric_motor.physical_systems import ConstantSpeedLoad\n", - "\n", - "load = 'ConstSpeedLoad'\n", - "load_initializer={'random_init': 'uniform', 'interval':[100,200] } \n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Reward Function\n", - "The reward calculation is based on the current state and reference of the motor environment. It is calculated as a weighted sum of errors with a certain power as follows:\n", - "\n", - "\n", - "\n", - "\n", - "$ r_t = - \\sum \\limits_{k=0} ^{N} w_{\\{k\\}}\\big | s_{\\{k\\}t} - s^*_{\\{k\\}t} \\big |^p $

\n", - "\n", - "Here, $r_t$ is the reward at time $t$. $ w_{\\{k\\}}$ is the reward weight for the k'th state variable of the $N$ referenced states . $s_{\\{k\\}t}$ is the k'th state variable of the motor and $s^*_{\\{k\\}t}$ is the corresponding k'th reference variable at time $t$. The reward power is $p$.\n", - "\n", - "\n", - "If states are observed for a constraint violation, an additional terminal reward is added. This value depends on the discount factor $\\gamma$ as follows. \n", - "\n", - "$r_t = -1 / (1 - \\gamma).$\n", - "\n", - "\n", - "### 3.1. Constraint Monitor\n", - "The constraint monitor observes the system states and assesses whether\n", - " they comply the given limits or violate them.\n", - " It returns the necessary information for the reward function, to calculate\n", - " the corresponding reward value.\n", - " The constraints of the system states can be generally described by the user\n", - " or is restricted by the physical enviroment limits as a default constraint.\n", - " \n", - " \n", - "The user defined constraint: \"SqdCurrentMonitor\" presented below, observes the currents and raises a flag indicating a limit violation if :\n", - "\n", - "$ i_{sd}^2 + i_{sq}^2 > i_{max}^2 $ \n", - "Here $i_{sd}$ and $i_{sq}$ are the motor currents in the d-q coordinate system that can be accessed from the motor states and $i_{max}$ is the maximum allowable current value.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "gamma = 0.99 #Discount factor for the reward punishment. Should equal agents' discount factor gamma.\n", - "\n", - "class SqdCurrentMonitor:\n", - " \"\"\"\n", - " monitor for squared currents:\n", - "\n", - " i_sd**2 + i_sq**2 < nominal_limit\n", - " \"\"\"\n", - "\n", - " def __call__(self, state, observed_states, k, physical_system):\n", - " self.I_SD_IDX = physical_system.state_names.index('i_sd') # access motor state i_sd\n", - " self.I_SQ_IDX = physical_system.state_names.index('i_sq') # access motor state i_sq\n", - " sqd_currents = state[self.I_SD_IDX] ** 2 + state[self.I_SQ_IDX] ** 2\n", - " return sqd_currents > 1 \n", - "\n", - " \n", - "reward_function=gem.reward_functions.WeightedSumOfErrors( # The function that computes the reward\n", - " observed_states=['i_sq', 'i_sd'], # Names of the observed states\n", - " reward_weights={'i_sq': 10, 'i_sd': 10}, # Reward power for each of the systems states.\n", - " constraint_monitor=SqdCurrentMonitor(), # ConstraintMonitor for monitoring\n", - " # states regarding defined constraints\n", - " gamma=gamma, # Discount factor for the reward punishment. Should equal agent's \n", - " # discount factor gamma.\n", - " reward_power=1) # Reward power for each of the systems states" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Reference Generator\n", - "The reference generator generates references to the observed states, that the physical system is expected to follow. GEM toolbox provides various reference generators which can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/reference_generators/reference_generator.html)\n", - "\n", - "In this example, references to the motor currents $i_{sq}$ and $i_{sd}$ are considered.
\n", - "The \"WienerProcessReferenceGenerator\" is used to generate random references for both $i_{sq}$ and $i_{sd}$. The wiener process is a stochastic process $W(t)$ for $t>=0$ with $W(0)=0$ and such that the increment $W(t)-W(s)$ is Gaussian with mean 0 and variance $\\sigma$ for any $0<=s\n", - "This example demonstrates the usage of the motor dashboard for visualization.
\n", - "A list of variables to be plotted is passed to the MotorDashboard during initialization. The variables that can be plotted for a given motor environment can be found in the [documentation.](https://upb-lea.github.io/gym-electric-motor/parts/visualizations/motor_dashboard.html)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from gym_electric_motor.visualization import MotorDashboard\n", - "\n", - "visualization = MotorDashboard(plots=['i_sq', 'i_sd', 'reward']) # plots the states i_sd and i_sq and reward." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Callbacks\n", - "GEM callbacks provide and easy to use interface to apply a set of functions on the motor environment during run time. Callbacks can be used to get a view of the internal states, collect statistics or modify certain motor parameters during runtime. \n", - "\n", - "GEM callbacks can be used to interact with the motor environment:\n", - "- At the start/end of every step\n", - "- At the start/end of every reset\n", - "- At the start of a close() call\n", - " \n", - "The following example provides a sample user defined callback implementation. The user defined callback object must be a sub-class of the 'Callback' class as shown. The objective of the user defined 'RewardLogger' callback object is to create a log of mean episode rewards of the experiment.\n", - "\n", - "Some of the interfaces implemented here are:\n", - "- __init() : A suitable class constructor.\n", - "- on_step_end() : A suitable task to be performed at the end of every step.\n", - "- on_reset_begin() : A suitable task at the beginning of each episode.\n", - "- on_close : A suitable task at the beginning of a close.\n", - "\n", - "For the list of all the interfaces, check out the GEM API documentation. \n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "from gym_electric_motor.core import Callback\n", - "import numpy as np\n", - "\n", - "class RewardLogger(Callback):\n", - " \"\"\" Logs the reward accumulated in each episode\n", - " \"\"\"\n", - " def __init__(self):\n", - " self._step_rewards = []\n", - " self._mean_episode_rewards = []\n", - "\n", - " def on_step_end(self):\n", - " \"\"\" stores the received reward at each step\n", - " \"\"\"\n", - " self._step_rewards.append(self._env._reward)\n", - " \n", - " def on_reset_begin(self):\n", - " \"\"\" stores the mean reward received in every episode\n", - " \"\"\"\n", - " self._mean_episode_rewards.append(np.mean(self._step_rewards))\n", - " self._step_rewards = []\n", - " \n", - " def on_close(self):\n", - " \"\"\" writes the mean episode reward of the experiment to a file.\n", - " \"\"\"\n", - " np.save(Path.cwd() /\"rl_frameworks\" / \"saved_agents\" / \"EpisodeRewards.npy\",\n", - " np.array(self._mean_episode_rewards))\n", - " \n", - "my_callback = [RewardLogger()] # instantiate the callback object " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. GEM Make Call\n", - "\n", - "The make function takes the environment-ids and several constructor arguments. Every environment also works without further parameters with default values. These default parameters can be looked up in the API-documentation of every GEM-environment [here](https://upb-lea.github.io/gym-electric-motor/index.html). \n", - "\n", - "The various components of the motor environment defined above are passed as arguements to the gem.make() function. This further returns the discrete PMSM motor environment.\n", - "\n", - "The motor environment can then be passed to a Reinforcement Learning agent in order to learn a controller." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "# define a PMSM with discrete action space\n", - "env = gem.make( \n", - " motor_env_id ,\n", - " # visualize the results\n", - " visualization=visualization,\n", - " # parameterize the PMSM and update limitations\n", - " motor_parameter=motor_parameter,\n", - " limit_values=limit_values, nominal_values=nominal_values,\n", - " # define the random initialisation for load and motor\n", - " load=load,\n", - " load_initializer=load_initializer,\n", - " motor_initializer=motor_initializer,\n", - " reward_function=reward_function,\n", - " tau=tau,\n", - " supply = supply,\n", - " supply_parameter=supply_parameter,\n", - " reference_generator=rg,\n", - " ode_solver='euler',\n", - " callbacks = my_callback\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "env" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.10" - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 4 -}