Skip to content

Commit

Permalink
RNN with dropout feature Part 1/2: API changes (#154)
Browse files Browse the repository at this point in the history 
* rnn api change for adding dropout feature
* initial rnn's dropout descriptor
* document RNN with dropout new APIs
* add descriptions for rnn v2 api
Co-authored-by: Daniel Lowell <[email protected]>
  • Loading branch information
@ce1adon
ce1adon authored Apr 17, 2020
1 parent ea37bd9 commit 574b070
Show file tree
Hide file tree
Showing 6 changed files with 439 additions and 38 deletions.
12 changes: 12 additions & 0 deletions doc/src/rnn.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ miopenGetRNNDescriptor
.. doxygenfunction:: miopenGetRNNDescriptor


miopenGetRNNDescriptor_V2
-------------------------

.. doxygenfunction:: miopenGetRNNDescriptor_V2


miopenDestroyRNNDescriptor
--------------------------

Expand All @@ -63,6 +69,12 @@ miopenSetRNNDescriptor
.. doxygenfunction:: miopenSetRNNDescriptor


miopenSetRNNDescriptor_V2
-------------------------

.. doxygenfunction:: miopenSetRNNDescriptor_V2


miopenGetRNNWorkspaceSize
-------------------------

Expand Down
176 changes: 163 additions & 13 deletions driver/rnn_driver.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include <fstream>
#include <memory>
#include <miopen/miopen.h>
#include <miopen/rnn.hpp>
#include <miopen/tensor.hpp>
#include <miopen/env.hpp>
#include <numeric>
Expand All @@ -64,6 +65,8 @@ class RNNDriver : public Driver
workspace_dev = nullptr;
reservespace_dev = nullptr;
data_type = (sizeof(Tgpu) == 4) ? miopenFloat : miopenHalf;

miopenCreateDropoutDescriptor(&DropoutDesc);
}

int AddCmdLineArgs();
Expand Down Expand Up @@ -122,6 +125,7 @@ class RNNDriver : public Driver
std::unique_ptr<GPUMem> dcy_dev;
std::unique_ptr<GPUMem> workspace_dev;
std::unique_ptr<GPUMem> reservespace_dev;
std::unique_ptr<GPUMem> dropout_states_dev;

std::vector<Tgpu> in;
std::vector<Tgpu> din;
Expand All @@ -148,13 +152,18 @@ class RNNDriver : public Driver
std::vector<Tref> cy_host;
std::vector<Tref> dhx_host;
std::vector<Tref> dcx_host;
std::vector<prngStates> dropout_states_host;

miopenRNNDescriptor_t rnnDesc;

int batchsize;
int adjustedSeqLen;
std::vector<int> batchseq;

miopenDropoutDescriptor_t DropoutDesc;
float dropout_rate;
unsigned long long dropout_seed;

// std::string GetVerificationCacheFileName() const;
// bool TryReadVerificationCache(const std::string& file_name,
// miopenTensorDescriptor_t& tensorDesc,
Expand Down Expand Up @@ -263,6 +272,14 @@ int RNNDriver<Tgpu, Tref>::AddCmdLineArgs()
"int");
inflags.AddInputFlag("datatype", 'f', "1", "16-bit or 32-bit fp (Default=1)", "int");

inflags.AddInputFlag(
"use_dropout", 'U', "0", "Use dropout: 1; Not use dropout: 0 (Default=0)", "int");
inflags.AddInputFlag("dropout", 'P', "0.0", "Dropout rate (Default=0.0)", "float");
inflags.AddInputFlag(
"seed_low", 'L', "0", "Least significant 32 bits of seed (Default=0)", "int");
inflags.AddInputFlag(
"seed_high", 'M', "0", "Most significant 32 bits of seed (Default=0)", "int");

return 0;
}

Expand Down Expand Up @@ -448,8 +465,56 @@ int RNNDriver<Tgpu, Tref>::SetRNNDescriptorFromCmdLineArgs()
exit(0);
}

miopenSetRNNDescriptor(
rnnDesc, wei_hh, layer, inMode, directionMode, mode, biasMode, algo, data_type);
if(inflags.GetValueInt("use_dropout"))
{
dropout_rate = static_cast<float>(inflags.GetValueDouble("dropout"));
auto dropout_seed_low =
static_cast<unsigned long long>(std::max(inflags.GetValueInt("seed_low"), 0));
auto dropout_seed_high =
static_cast<unsigned long long>(std::max(inflags.GetValueInt("seed_high"), 0));
dropout_seed = dropout_seed_high << 32 | dropout_seed_low;

size_t statesSizeInBytes = 0;
miopenDropoutGetStatesSize(GetHandle(), &statesSizeInBytes);
size_t states_size = statesSizeInBytes / sizeof(prngStates);

#if MIOPEN_BACKEND_OPENCL
cl_context ctx;

clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr);
#elif MIOPEN_BACKEND_HIP
uint32_t ctx = 0;
#endif

dropout_states_dev =
std::unique_ptr<GPUMem>(new GPUMem(ctx, states_size, sizeof(prngStates)));

miopenSetDropoutDescriptor(DropoutDesc,
GetHandle(),
dropout_rate,
dropout_states_dev->GetMem(),
dropout_states_dev->GetSize(),
dropout_seed,
false,
false,
MIOPEN_RNG_PSEUDO_XORWOW);

miopenSetRNNDescriptor_V2(rnnDesc,
wei_hh,
layer,
DropoutDesc,
inMode,
directionMode,
mode,
biasMode,
algo,
data_type);
}
else
{
miopenSetRNNDescriptor(
rnnDesc, wei_hh, layer, inMode, directionMode, mode, biasMode, algo, data_type);
}

return miopenStatusSuccess;
}
Expand Down Expand Up @@ -497,7 +562,7 @@ int RNNDriver<Tgpu, Tref>::AllocateBuffersAndCopy()

clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ctx, nullptr);
#elif MIOPEN_BACKEND_HIP
uint32_t ctx = 0;
uint32_t ctx = 0;
#endif

in_dev = std::unique_ptr<GPUMem>(new GPUMem(ctx, in_sz, sizeof(Tgpu)));
Expand Down Expand Up @@ -848,9 +913,26 @@ int RNNDriver<Tgpu, Tref>::RunForwardCPU()
miopenRNNDirectionMode_t dirMode;
miopenRNNBiasMode_t biasMode;
int hiddenSize;
miopenDropoutDescriptor_t drop_desc;

miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
if(inflags.GetValueInt("use_dropout"))
{
miopenGetRNNDescriptor_V2(rnnDesc,
&hiddenSize,
&layer,
&drop_desc,
&inputMode,
&dirMode,
&mode,
&biasMode,
&algoMode,
nullptr);
}
else
{
miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
}

bidirection = (dirMode == miopenRNNbidirection);
biased = (biasMode == miopenRNNwithBias);
Expand Down Expand Up @@ -1101,9 +1183,26 @@ int RNNDriver<Tgpu, Tref>::RunBackwardWeightsCPU()
miopenRNNDirectionMode_t dirMode;
miopenRNNBiasMode_t biasMode;
int hiddenSize;
miopenDropoutDescriptor_t drop_desc;

miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
if(inflags.GetValueInt("use_dropout"))
{
miopenGetRNNDescriptor_V2(rnnDesc,
&hiddenSize,
&layer,
&drop_desc,
&inputMode,
&dirMode,
&mode,
&biasMode,
&algoMode,
nullptr);
}
else
{
miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
}

bidirection = (dirMode == miopenRNNbidirection);
biased = (biasMode == miopenRNNwithBias);
Expand Down Expand Up @@ -1210,9 +1309,26 @@ int RNNDriver<Tgpu, Tref>::RunBackwardDataCPU()
miopenRNNDirectionMode_t dirMode;
miopenRNNBiasMode_t biasMode;
int hiddenSize;
miopenDropoutDescriptor_t drop_desc;

miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
if(inflags.GetValueInt("use_dropout"))
{
miopenGetRNNDescriptor_V2(rnnDesc,
&hiddenSize,
&layer,
&drop_desc,
&inputMode,
&dirMode,
&mode,
&biasMode,
&algoMode,
nullptr);
}
else
{
miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
}

bidirection = (dirMode == miopenRNNbidirection);
biased = (biasMode == miopenRNNwithBias);
Expand Down Expand Up @@ -1396,9 +1512,26 @@ int RNNDriver<Tgpu, Tref>::VerifyForward()
miopenRNNDirectionMode_t dirMode;
miopenRNNBiasMode_t biasMode;
int hiddenSize;
miopenDropoutDescriptor_t drop_desc;

miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
if(inflags.GetValueInt("use_dropout"))
{
miopenGetRNNDescriptor_V2(rnnDesc,
&hiddenSize,
&layer,
&drop_desc,
&inputMode,
&dirMode,
&mode,
&biasMode,
&algoMode,
nullptr);
}
else
{
miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
}

if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode))
{
Expand Down Expand Up @@ -1469,9 +1602,26 @@ int RNNDriver<Tgpu, Tref>::VerifyBackward()
miopenRNNDirectionMode_t dirMode;
miopenRNNBiasMode_t biasMode;
int hiddenSize;
miopenDropoutDescriptor_t drop_desc;

miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
if(inflags.GetValueInt("use_dropout"))
{
miopenGetRNNDescriptor_V2(rnnDesc,
&hiddenSize,
&layer,
&drop_desc,
&inputMode,
&dirMode,
&mode,
&biasMode,
&algoMode,
nullptr);
}
else
{
miopenGetRNNDescriptor(
rnnDesc, &mode, &algoMode, &inputMode, &dirMode, &biasMode, &hiddenSize, &layer);
}

if(CheckGuard(in_h, out_h, hy_d, hy_n, hy_h, dirMode, inputMode))
{
Expand Down
56 changes: 56 additions & 0 deletions include/miopen/miopen.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3043,6 +3043,33 @@ MIOPEN_EXPORT miopenStatus_t miopenGetRNNDescriptor(miopenRNNDescriptor_t rnnDes
int* hiddenSize,
int* layer);

/*! @brief Retrieves a RNN layer descriptor's details version 2. This version enables retrieving
* information of the dropout descriptor of the rnn descriptor.
*
* @param rnnDesc RNN layer descriptor (input)
* @param hiddenSize Size of hidden state (output)
* @param layer Number of stacked layers (output)
* @param dropoutDesc Pre-configured dropout descriptor for dropout layer in between RNN layers
* (output)
* @param inputMode RNN data input mode (output)
* @param dirMode Uni or bi direction mode (output)
* @param rnnMode RNN mode (output)
* @param biasMode Bias used (output)
* @param algoMode RNN algorithm mode (output)
* @param dataType Data type of RNN (output)
* @return miopenStatus_t
*/
MIOPEN_EXPORT miopenStatus_t miopenGetRNNDescriptor_V2(miopenRNNDescriptor_t rnnDesc,
int* hiddenSize,
int* layer,
miopenDropoutDescriptor_t* dropoutDesc,
miopenRNNInputMode_t* inputMode,
miopenRNNDirectionMode_t* dirMode,
miopenRNNMode_t* rnnMode,
miopenRNNBiasMode_t* biasMode,
miopenRNNAlgo_t* algoMode,
miopenDataType_t* dataType);

/*! @brief Destroys the tensor descriptor object
*
* @param rnnDesc RNN tensor descriptor type (input)
Expand Down Expand Up @@ -3075,6 +3102,35 @@ MIOPEN_EXPORT miopenStatus_t miopenSetRNNDescriptor(miopenRNNDescriptor_t rnnDes
miopenRNNAlgo_t algo,
miopenDataType_t dataType);

/*! @brief Set the details of the RNN descriptor version 2. This version enables the use of dropout
* in rnn.
*
* Interface for setting the values of the RNN descriptor object. This function requires specific
* algorithm selection.
* @param rnnDesc RNN layer descriptor type (input/output)
* @param hsize Hidden layer size (input)
* @param nlayers Number of layers (input)
* @param dropoutDesc Pre-initialized dropout descriptor for dropout layer in between RNN layers
* (input)
* @param inMode RNN first layer input mode (input)
* @param direction RNN direction (input)
* @param rnnMode RNN model type (input)
* @param biasMode RNN bias included (input)
* @param algo RNN algorithm selected (input)
* @param dataType Only fp32 currently supported for RNNs (input)
* @return miopenStatus_t
*/
MIOPEN_EXPORT miopenStatus_t miopenSetRNNDescriptor_V2(miopenRNNDescriptor_t rnnDesc,
const int hsize,
const int nlayers,
miopenDropoutDescriptor_t dropoutDesc,
miopenRNNInputMode_t inMode,
miopenRNNDirectionMode_t direction,
miopenRNNMode_t rnnMode,
miopenRNNBiasMode_t biasMode,
miopenRNNAlgo_t algo,
miopenDataType_t dataType);

/*! @brief Query the amount of memory required to execute the RNN layer
*
* This function calculates the amount of memory required to run the RNN layer given an RNN
Expand Down
Loading

0 comments on commit 574b070

Please sign in to comment.