forked from Azure/usql
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added example job to demonstrate Avro extractor usage
- Loading branch information
Showing
8 changed files
with
167 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
|
|
||
| Microsoft Visual Studio Solution File, Format Version 12.00 | ||
| # Visual Studio 14 | ||
| VisualStudioVersion = 14.0.25420.1 | ||
| MinimumVisualStudioVersion = 10.0.40219.1 | ||
| Project("{182E2583-ECAD-465B-BB50-91101D7C24CE}") = "AvroExamples", "AvroExamples\AvroExamples.usqlproj", "{3013DB4A-4F98-4EE0-9796-3172FB40C065}" | ||
| EndProject | ||
| Global | ||
| GlobalSection(SolutionConfigurationPlatforms) = preSolution | ||
| Debug|Any CPU = Debug|Any CPU | ||
| Debug|x64 = Debug|x64 | ||
| Debug|x86 = Debug|x86 | ||
| Release|Any CPU = Release|Any CPU | ||
| Release|x64 = Release|x64 | ||
| Release|x86 = Release|x86 | ||
| EndGlobalSection | ||
| GlobalSection(ProjectConfigurationPlatforms) = postSolution | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|Any CPU.ActiveCfg = Debug|Any CPU | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|Any CPU.Build.0 = Debug|Any CPU | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|x64.ActiveCfg = Debug|x64 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|x64.Build.0 = Debug|x64 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|x86.ActiveCfg = Debug|x86 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Debug|x86.Build.0 = Debug|x86 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|Any CPU.ActiveCfg = Release|Any CPU | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|Any CPU.Build.0 = Release|Any CPU | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|x64.ActiveCfg = Release|x64 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|x64.Build.0 = Release|x64 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|x86.ActiveCfg = Release|x86 | ||
| {3013DB4A-4F98-4EE0-9796-3172FB40C065}.Release|x86.Build.0 = Release|x86 | ||
| EndGlobalSection | ||
| GlobalSection(SolutionProperties) = preSolution | ||
| HideSolutionNode = FALSE | ||
| EndGlobalSection | ||
| EndGlobal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| DROP DATABASE IF EXISTS Avro; | ||
|
|
||
| CREATE DATABASE Avro; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| using Microsoft.Analytics.Interfaces; | ||
| using Microsoft.Analytics.Types.Sql; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Text; | ||
|
|
||
| namespace AvroExamples | ||
| { | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| DROP ASSEMBLY IF EXISTS [Microsoft.Hadoop.Avro]; | ||
| CREATE ASSEMBLY [Microsoft.Hadoop.Avro] FROM @"/Assemblies/Avro/Microsoft.Hadoop.Avro.dll"; | ||
| DROP ASSEMBLY IF EXISTS [Microsoft.Analytics.Samples.Formats]; | ||
| CREATE ASSEMBLY [Microsoft.Analytics.Samples.Formats] FROM @"/Assemblies/Avro/Microsoft.Analytics.Samples.Formats.dll"; | ||
| DROP ASSEMBLY IF EXISTS [Newtonsoft.Json]; | ||
| CREATE ASSEMBLY [Newtonsoft.Json] FROM @"/Assemblies/Avro/Newtonsoft.Json.dll"; |
12 changes: 12 additions & 0 deletions
12
Examples/AvroExamples/AvroExamples/2-RegisterAssemblies.usql.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| using Microsoft.Analytics.Interfaces; | ||
| using Microsoft.Analytics.Types.Sql; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Text; | ||
|
|
||
| namespace AvroExamples | ||
| { | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| REFERENCE ASSEMBLY [Newtonsoft.Json]; | ||
| REFERENCE ASSEMBLY [Microsoft.Hadoop.Avro]; | ||
| REFERENCE ASSEMBLY [Microsoft.Analytics.Samples.Formats]; | ||
|
|
||
| DECLARE @input_file string = @"\TwitterStream\{*}\{*}\{*}.avro"; | ||
| DECLARE @output_file string = @"\output\twitter.csv"; | ||
|
|
||
| @rs = | ||
| EXTRACT | ||
| createdat string, | ||
| topic string, | ||
| sentimentscore long, | ||
| eventprocessedutctime string, | ||
| partitionid long, | ||
| eventenqueuedutctime string | ||
| FROM @input_file | ||
| USING new Microsoft.Analytics.Samples.Formats.Avro.AvroExtractor(@" | ||
| { | ||
| ""type"" : ""record"", | ||
| ""name"" : ""GenericFromIRecord0"", | ||
| ""namespace"" : ""Microsoft.Streaming.Avro"", | ||
| ""fields"" : [ { | ||
| ""name"" : ""createdat"", | ||
| ""type"" : [ ""null"", ""string"" ] | ||
| }, { | ||
| ""name"" : ""topic"", | ||
| ""type"" : [ ""null"", ""string"" ] | ||
| }, { | ||
| ""name"" : ""sentimentscore"", | ||
| ""type"" : [ ""null"", ""long"" ] | ||
| }, { | ||
| ""name"" : ""eventprocessedutctime"", | ||
| ""type"" : [ ""null"", ""string"" ] | ||
| }, { | ||
| ""name"" : ""partitionid"", | ||
| ""type"" : [ ""null"", ""long"" ] | ||
| }, { | ||
| ""name"" : ""eventenqueuedutctime"", | ||
| ""type"" : [ ""null"", ""string"" ] | ||
| } ] | ||
| } | ||
| "); | ||
|
|
||
| @cnt = | ||
| SELECT topic, COUNT(*) AS cnt | ||
| FROM @rs | ||
| GROUP BY topic; | ||
|
|
||
| OUTPUT @cnt TO @output_file USING Outputters.Text(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| using Microsoft.Analytics.Interfaces; | ||
| using Microsoft.Analytics.Types.Sql; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Linq; | ||
| using System.Text; | ||
|
|
||
| namespace AvroExamples | ||
| { | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| <?xml version="1.0" encoding="utf-8"?> | ||
| <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> | ||
| <PropertyGroup> | ||
| <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> | ||
| <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> | ||
| <SchemaVersion>2.0</SchemaVersion> | ||
| <ProjectGuid>3013db4a-4f98-4ee0-9796-3172fb40c065</ProjectGuid> | ||
| <OutputType>File</OutputType> | ||
| <AssemblyName>Algebra.xml</AssemblyName> | ||
| <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion> | ||
| <Name>AvroExamples</Name> | ||
| <RootNamespace>AvroExamples</RootNamespace> | ||
| <RuntimeVersion>default</RuntimeVersion> | ||
| <OutputStreamPath>C:\Users\flmader\AppData\Local\USQLDataRoot</OutputStreamPath> | ||
| </PropertyGroup> | ||
| <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> | ||
| <DebugSymbols>true</DebugSymbols> | ||
| <OutputPath>bin\Debug\</OutputPath> | ||
| </PropertyGroup> | ||
| <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> | ||
| <DebugSymbols>false</DebugSymbols> | ||
| <OutputPath>bin\Release\</OutputPath> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <Script Include="1-CreateDB.usql" /> | ||
| <Script Include="2-RegisterAssemblies.usql" /> | ||
| <Script Include="3-SimpleAvro.usql" /> | ||
| <ScriptCode Include="1-CreateDB.usql.cs"> | ||
| <DependentUpon>1-CreateDB.usql</DependentUpon> | ||
| </ScriptCode> | ||
| <ScriptCode Include="2-RegisterAssemblies.usql.cs"> | ||
| <DependentUpon>2-RegisterAssemblies.usql</DependentUpon> | ||
| </ScriptCode> | ||
| <ScriptCode Include="3-SimpleAvro.usql.cs"> | ||
| <DependentUpon>3-SimpleAvro.usql</DependentUpon> | ||
| </ScriptCode> | ||
| </ItemGroup> | ||
| <Import Project="$(AppData)\Microsoft\DataLake\MsBuild\1.0\Usql.targets" /> | ||
| </Project> |