Skip to content

Commit

Permalink
BrainScript:
Browse files Browse the repository at this point in the history 
added pretty-printed expression names for infix operators;
the ":" operator (for forming arrays) now flattens the array in the parser rather than the evaluation, allowing evaluation to construct a ConfigArray where the elements are lazily evaluated, as needed for the top-level "actions";
added the parsing and evaluation of the (dict with dict) syntax, although its functionality is not yet implemented (just returns the first dict);
disabled the 'stopAtNewline' flag, since I don't see why it is even necessary, and it prevents us from writing the "with" operator on the next line;
ParseConfigExpression() now checks for junk at end;
wmainWithBS() now parses BS with ParseConfigExpression() rather than ParseConfigDictFromString(), so that we can construct a "with" expression with overrides given on the command line
  • Loading branch information
@frankseide
frankseide committed Nov 21, 2015
1 parent 280f1e8 commit 34c76da
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 51 deletions.
74 changes: 41 additions & 33 deletions BrainScript/BrainScriptEvaluator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -881,13 +881,14 @@ namespace Microsoft { namespace MSR { namespace BS {
typedef function<ConfigValuePtr(const ExpressionPtr & e, ConfigValuePtr leftVal, ConfigValuePtr rightVal, const IConfigRecordPtr & scope, const wstring & exprPath)> InfixOp /*const*/;
struct InfixOps
{
InfixOp NumbersOp; // number OP number -> number
InfixOp StringsOp; // string OP string -> string
InfixOp BoolOp; // bool OP bool -> bool
InfixOp ComputeNodeOp; // one operand is ComputeNode -> ComputeNode
InfixOp DictOp; // dict OP dict
InfixOps(InfixOp NumbersOp, InfixOp StringsOp, InfixOp BoolOp, InfixOp ComputeNodeOp, InfixOp DictOp)
: NumbersOp(NumbersOp), StringsOp(StringsOp), BoolOp(BoolOp), ComputeNodeOp(ComputeNodeOp), DictOp(DictOp) { }
wstring prettyName; // pretty-printable name of this op, e.g. "Plus" for +
InfixOp NumbersOp; // number OP number -> number
InfixOp StringsOp; // string OP string -> string
InfixOp BoolOp; // bool OP bool -> bool
InfixOp ComputeNodeOp; // one operand is ComputeNode -> ComputeNode
InfixOp DictOp; // dict OP dict
InfixOps(const wchar_t * name, InfixOp NumbersOp, InfixOp StringsOp, InfixOp BoolOp, InfixOp ComputeNodeOp, InfixOp DictOp)
: prettyName(name), NumbersOp(NumbersOp), StringsOp(StringsOp), BoolOp(BoolOp), ComputeNodeOp(ComputeNodeOp), DictOp(DictOp) { }
};

// functions that implement infix operations
Expand Down Expand Up @@ -1000,29 +1001,39 @@ namespace Microsoft { namespace MSR { namespace BS {
valueWithName->SetName(value.GetExpressionName());
return value;
};
static ConfigValuePtr DictOp(const ExpressionPtr & e, ConfigValuePtr leftVal, ConfigValuePtr rightVal, const IConfigRecordPtr & scope, const wstring & exprPath)
{
if(e->op != L"with")
LogicError("unexpected infix op");
let left = leftVal.AsPtr<ConfigRecord>();
let right = rightVal.AsPtr<ConfigRecord>();
left; right; scope; exprPath; // TODO: create a composite dictionary
return leftVal;
};
static ConfigValuePtr BadOp(const ExpressionPtr & e, ConfigValuePtr, ConfigValuePtr, const IConfigRecordPtr &, const wstring &) { InvalidInfixOpTypes(e); };

// lookup table for infix operators
// This lists all infix operators with lambdas for evaluating them.
static map<wstring, InfixOps> infixOps =
{
// NumbersOp StringsOp BoolOp ComputeNodeOp DictOp TODO: this comment is incomplete
{ L"*", InfixOps(NumOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"/", InfixOps(NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L".*", InfixOps(BadOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"**", InfixOps(NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L"%", InfixOps(NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L"+", InfixOps(NumOp, StrOp, BadOp, NodeOp, BadOp) },
{ L"-", InfixOps(NumOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"==", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"!=", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"<", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L">", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"<=", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L">=", InfixOps(NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"&&", InfixOps(BadOp, BadOp, BoolOp, BadOp, BadOp) },
{ L"||", InfixOps(BadOp, BadOp, BoolOp, BadOp, BadOp) },
{ L"^", InfixOps(BadOp, BadOp, BoolOp, BadOp, BadOp) }
// PrettyName NumbersOp StringsOp BoolOp ComputeNodeOp DictOp
{ L"with", InfixOps(L"Times", NumOp, BadOp, BadOp, NodeOp, DictOp) },
{ L"*", InfixOps(L"Times", NumOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"/", InfixOps(L"Div", NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L".*", InfixOps(L"DotTimes", BadOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"**", InfixOps(L"Pow", NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L"%", InfixOps(L"Mod", NumOp, BadOp, BadOp, BadOp, BadOp) },
{ L"+", InfixOps(L"Plus", NumOp, StrOp, BadOp, NodeOp, BadOp) },
{ L"-", InfixOps(L"Minus", NumOp, BadOp, BadOp, NodeOp, BadOp) },
{ L"==", InfixOps(L"Equal", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"!=", InfixOps(L"NotEqual", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"<", InfixOps(L"LT", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L">", InfixOps(L"GT", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"<=", InfixOps(L"LE", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L">=", InfixOps(L"GT", NumOp, StrOp, BoolOp, BadOp, BadOp) },
{ L"&&", InfixOps(L"And", BadOp, BadOp, BoolOp, BadOp, BadOp) },
{ L"||", InfixOps(L"Or", BadOp, BadOp, BoolOp, BadOp, BadOp) },
{ L"^", InfixOps(L"Xor", BadOp, BadOp, BoolOp, BadOp, BadOp) }
};

// -----------------------------------------------------------------------
Expand Down Expand Up @@ -1255,11 +1266,7 @@ namespace Microsoft { namespace MSR { namespace BS {
for (size_t i = 0; i < e->args.size(); i++) // concatenate the two args
{
let & expr = e->args[i];
let item = Evaluate(expr, scope, exprPath, wstrprintf(L"[%d]", i)); // result can be an item or a vector
if (item.Is<ConfigArray>())
arr->Append(item.AsRef<ConfigArray>()); // append all elements (this flattens it)
else
arr->Append(item);
arr->Append(move(MakeEvaluateThunkPtr(expr, scope, msra::strfun::wstrprintf(L"%ls[%d]", exprPath.c_str(), i), L"")));
}
return ConfigValuePtr(arr, MakeFailFn(e->location), exprPath); // location will be that of the first ':', not sure if that is best way
}
Expand Down Expand Up @@ -1333,12 +1340,12 @@ namespace Microsoft { namespace MSR { namespace BS {
{
let opIter = infixOps.find(e->op);
if (opIter == infixOps.end())
LogicError("e->op " + utf8(e->op) + " not implemented");
LogicError("e->op '%ls' not implemented", e->op.c_str());
let & functions = opIter->second;
let & leftArg = e->args[0];
let & rightArg = e->args[1];
let leftValPtr = Evaluate(leftArg, scope, exprPath, L"/*" + e->op + L"*/left");
let rightValPtr = Evaluate(rightArg, scope, exprPath, L"/*" + e->op + L"*/right");
let leftValPtr = Evaluate(leftArg, scope, exprPath, functions.prettyName + L"_left");
let rightValPtr = Evaluate(rightArg, scope, exprPath, functions.prettyName + L"_right");
if (leftValPtr.Is<Double>() && rightValPtr.Is<Double>())
return functions.NumbersOp(e, leftValPtr, rightValPtr, scope, exprPath);
else if (leftValPtr.Is<String>() && rightValPtr.Is<String>())
Expand All @@ -1352,7 +1359,8 @@ namespace Microsoft { namespace MSR { namespace BS {
return functions.ComputeNodeOp(e, leftValPtr, rightValPtr, scope, exprPath);
else if (leftValPtr.Is<Double>() && rightValPtr.Is<ComputationNodeObject>())
return functions.ComputeNodeOp(e, leftValPtr, rightValPtr, scope, exprPath);
// TODO: DictOp --maybe not; maybedo this in ModelMerger class instead
else if (leftValPtr.Is<ConfigRecord>() && rightValPtr.Is<ConfigRecord>())
return functions.DictOp(e, leftValPtr, rightValPtr, scope, exprPath);
else
InvalidInfixOpTypes(e);
}
Expand Down
40 changes: 34 additions & 6 deletions BrainScript/BrainScriptParser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ class Lexer : public CodeSource
keywords = set<wstring>
{
L"include",
L"new", L"true", L"false",
L"new", L"with", L"true", L"false",
L"if", L"then", L"else",
L"array",
};
Expand Down Expand Up @@ -537,6 +537,7 @@ class Parser : public Lexer
{ L".", 100 }, { L"[", 100 }, { L"(", 100 }, // also sort-of infix operands...
{ L"*", 10 }, { L"/", 10 }, { L".*", 10 }, { L"**", 10 }, { L"%", 10 },
{ L"+", 9 }, { L"-", 9 },
{ L"with", 9 },
{ L"==", 8 }, { L"!=", 8 }, { L"<", 8 }, { L"<=", 8 }, { L">", 8 }, { L">=", 8 },
{ L"&&", 7 },
{ L"||", 6 },
Expand Down Expand Up @@ -633,8 +634,13 @@ class Parser : public Lexer
for (;;)
{
let & opTok = GotToken();
if (stopAtNewline && opTok.isLineInitial)
break;
// BUGBUG: 'stopAtNewline' is broken.
// It does not prevent "a = 13 b = 42" from being accepted.
// On the other hand, it would prevent the totally valid "dict \n with dict2".
// A correct solution should require "a = 13 ; b = 42", i.e. a semicolon or newline,
// while continuing to parse across newlines when syntactically meaningful (there is no ambiguity in BrainScript).
//if (stopAtNewline && opTok.isLineInitial)
// break;
let opIter = infixPrecedence.find(opTok.symbol);
if (opIter == infixPrecedence.end()) // not an infix operator: we are done here, 'left' is our expression
break;
Expand Down Expand Up @@ -675,6 +681,18 @@ class Parser : public Lexer
operation->args.push_back(ParseExpression(0, false)); // [1]: index
ConsumePunctuation(L"]");
}
else if (op == L":")
{
// special case: (a : b : c) gets flattened into :(a,b,c) i.e. an operation with possibly >2 operands
ConsumeToken();
let right = ParseExpression(opPrecedence + 1, stopAtNewline); // get right operand, or entire multi-operand expression with higher precedence
if (left->op == L":") // appending to a list: flatten it
{
operation->args = left->args;
operation->location = left->location; // location of first ':' (we need to choose some location)
}
operation->args.push_back(right); // form a list of multiple operands (not just two)
}
else // === regular infix operator
{
ConsumeToken();
Expand Down Expand Up @@ -789,12 +807,16 @@ class Parser : public Lexer
}
return members;
}
void VerifyAtEnd()
{
if (GotToken().kind != eof)
Fail(L"junk at end of source", GetCursor());
}
// top-level parse function parses dictonary members without enclosing [ ... ] and returns it as a dictionary
ExpressionPtr ParseRecordMembersToDict()
{
let topMembers = ParseRecordMembers();
if (GotToken().kind != eof)
Fail(L"junk at end of source", GetCursor());
VerifyAtEnd();
ExpressionPtr topDict = make_shared<Expression>(GetCursor(), L"[]");
topDict->namedArgs = topMembers;
return topDict;
Expand All @@ -811,6 +833,12 @@ class Parser : public Lexer
static ExpressionPtr Parse(SourceFile && sourceFile, vector<wstring> && includePaths) { return Parser(move(sourceFile), move(includePaths)).ParseRecordMembersToDict(); }
ExpressionPtr ParseConfigDictFromString(wstring text, vector<wstring> && includePaths) { return Parse(SourceFile(L"(command line)", text), move(includePaths)); }
ExpressionPtr ParseConfigDictFromFile(wstring path, vector<wstring> && includePaths) { auto sourceFile = SourceFile(path, includePaths); return Parse(move(sourceFile), move(includePaths)); }
ExpressionPtr ParseConfigExpression(const wstring & sourceText, vector<wstring> && includePaths) { return Parser(SourceFile(L"(command line)", sourceText), move(includePaths)).ParseExpression(0, true/*can end at newline*/); }
ExpressionPtr ParseConfigExpression(const wstring & sourceText, vector<wstring> && includePaths)
{
auto parser = Parser(SourceFile(L"(command line)", sourceText), move(includePaths));
auto expr = parser.ParseExpression(0, true/*can end at newline*/);
parser.VerifyAtEnd();
return expr;
}

}}} // namespaces
6 changes: 4 additions & 2 deletions Common/Include/ScriptableObjects.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,9 +453,11 @@ namespace Microsoft { namespace MSR { namespace ScriptableObjects {
public:
ConfigArray() : firstIndex(0) { }
ConfigArray(int firstIndex, vector<ConfigValuePtr> && values) : firstIndex(firstIndex), values(move(values)) { }
pair<int, int> GetIndexRange() const { return make_pair(firstIndex, firstIndex+(int)values.size()-1); }
//ConfigArray(ConfigValuePtr && val) : firstIndex(0), values(vector<ConfigValuePtr>{ move(val) }) { }
pair<int, int> GetIndexRange() const { return make_pair(firstIndex, firstIndex + (int)values.size() - 1); }
// building the array from expressions: append an element or an array
void Append(ConfigValuePtr value) { values.push_back(value); }
void Append(const ConfigValuePtr & value) { values.push_back(value); }
void Append(ConfigValuePtr && value) { values.push_back(move(value)); } // this appends an unresolved ConfigValuePtr
void Append(const ConfigArray & other) { values.insert(values.end(), other.values.begin(), other.values.end()); }
// get element at index, including bounds check
template<typename FAILFN>
Expand Down
28 changes: 18 additions & 10 deletions MachineLearning/CNTK/CNTK.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1589,20 +1589,19 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra
_wchdir(workingDir.c_str());

// compile the BrainScript
wstring bs;
wstring bs = L"[\n";
bs += standardFunctions + computationNodes + commonMacros + L"\n"; // start with standard macros
for (const auto & sourceFile : sourceFiles)
bs += L"include " + PathToBSStringLiteral(sourceFile) + L"\n";
#if 0
bs += L"\n]\n";
for (const auto & over : overrides)
bs += L"with [ " + over + L" ]\n";
#endif

fprintf(stderr, "\n\nBrainScript -->\n\n%ls\n\n", bs.c_str());

let expr = BS::ParseConfigDictFromString(bs, move(includePaths)); // parse --TODO: support include path
let valp = BS::Evaluate(expr); // evaluate parse into a dictionary
let & config = valp.AsRef<ScriptableObjects::IConfigRecord>(); // this is the dictionary
let expr = BS::ParseConfigExpression(bs, move(includePaths)); // parse
let valp = BS::Evaluate(expr); // evaluate parse into a dictionary
let & config = valp.AsRef<ScriptableObjects::IConfigRecord>(); // this is the dictionary

// legacy parameters that have changed spelling
if (config.Find(L"DoneFile")) // variables follow camel case (start with lower-case letters)
Expand Down Expand Up @@ -1656,11 +1655,20 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wra
ProgressTracing::TraceTotalNumberOfSteps(fullTotalMaxEpochs); // enable tracing, using this as the total number of epochs

// MAIN LOOP that executes the actions
const ScriptableObjects::ConfigArray & actions = config[L"actions"];
for (int i = actions.GetIndexRange().first; i <= actions.GetIndexRange().second; i++)
{
actions.At(i, [](const wstring &){}); // this will evaluate and thus execute the action
auto actionsVal = config[L"actions"];
// Note: weird behavior. If 'actions' is a single value (rather than an array) then it will have been resolved already. That means, it has already completed the action.
// Not pretty, but a direct consequence of the lazy evaluation. The only good solution would be to have a syntax for arrays including length 0 and 1.
// Since this in the end behaves indistinguishable from the array loop below, we will keep it for now.
if (actionsVal.Is<ScriptableObjects::ConfigArray>())
{
const ScriptableObjects::ConfigArray & actions = actionsVal.AsRef<ScriptableObjects::ConfigArray>();
// Note: We must use AsRef<>() here. Just assigning (using the auto-typecast) will make a copy, which will ^^ not work since the elements are not yet resolved.
for (int i = actions.GetIndexRange().first; i <= actions.GetIndexRange().second; i++)
{
actions.At(i, [](const wstring &){}); // this will evaluate and thus execute the action
}
}
// else action has already been executed, see comment above

// write a doneFile if requested
wstring doneFile = config(L"doneFile", L"");
Expand Down

0 comments on commit 34c76da

Please sign in to comment.