Change clog to use a published nuget package (microsoft#612)

* Change clog to use a published nuget package

Much cleaner and easier to update. The executable now contains all the clog dependencies, and extracts them when we run --installDependencies

.azure/templates/build-config-user.yml

@@ -22,6 +22,14 @@ jobs:
     ${{ if eq(parameters.platform, 'windows') }}:
       path: msquic
 
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    condition: eq('${{ parameters.platform }}', 'linux')
+    inputs:
+      packageType: sdk
+      version: 3.1.302
+      installationPath: $(Agent.ToolsDirectory)/dotnet
+
   - task: ShellScript@2
     displayName: Install Powershell
     condition: eq('${{ parameters.platform }}', 'linux')

.azure/templates/download-artifacts.yml

@@ -22,13 +22,6 @@ steps:
       itemPattern: bin/winkernel/${{ parameters.arch }}_*_${{ parameters.tls }}/**
       downloadPath: artifacts
 
-- task: DownloadBuildArtifacts@0
-  displayName: Download CLOG Artifacts
-  inputs:
-    artifactName: bin
-    itemPattern: bin/clog/**
-    downloadPath: artifacts
-
 - task: PowerShell@2
   displayName: Install Build Artifacts
   inputs:

.dockerignore

@@ -2,7 +2,7 @@
 !CMakeLists.txt
 !scripts
 !src
-!submodules/clog
+!cmake
 !submodules/openssl
 submodules/openssl/pyca-cryptography
 submodules/openssl/boringssl

.gitmodules

@@ -12,6 +12,3 @@
 [submodule "submodules/wil"]
 	path = submodules/wil
 	url = https://github.com/microsoft/wil
-[submodule "submodules/clog"]
-	path = submodules/clog
-	url = https://github.com/microsoft/CLOG.git

CMakeLists.txt

@@ -140,6 +140,8 @@ set(QUIC_OUTPUT_DIR ${QUIC_BUILD_DIR}/bin/$<IF:$<CONFIG:Debug>,Debug,Release> CA
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${QUIC_BUILD_DIR}/obj/$<IF:$<CONFIG:Debug>,Debug,Release>)
 
+execute_process(COMMAND clog --installDirectory ${CMAKE_SOURCE_DIR}/build/clog)
+
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${QUIC_OUTPUT_DIR})
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${QUIC_OUTPUT_DIR})
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_RELEASE ${QUIC_OUTPUT_DIR})
@@ -148,18 +150,13 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY_DEBUG ${QUIC_OUTPUT_DIR})
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${QUIC_OUTPUT_DIR})
 
 set(CMAKE_CLOG_OUTPUT_DIRECTORY ${QUIC_BUILD_DIR}/inc)
-set(CMAKE_CLOG_BINS_DIRECTORY ${QUIC_BUILD_DIR}/submodules/clog)
 set(CMAKE_CLOG_SIDECAR_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/manifest)
-set(CLOG_SOURCE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/submodules/clog)
-set(CLOG_INCLUDE_DIRECTORY ${CLOG_SOURCE_DIRECTORY}/inc)
+set(CLOG_INCLUDE_DIRECTORY ${CMAKE_SOURCE_DIR}/build/clog)
+set(CMAKE_CLOG_GENERATE_FILE ${CMAKE_SOURCE_DIR}/build/clog/CLog.cmake)
 set(CMAKE_CLOG_CONFIG_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/manifest/msquic.clog_config)
 
 set(MANIFESTED_HEADER_FILENAME ${QUIC_BUILD_DIR}/inc/MsQuicEtw.h)
 
-add_custom_target(CLOG_GENERATED_FILES
-    COMMENT "CREATED CMAKE Target for CLOG generated files"
-)
-
 set(CLANG_GCC_WARNING_FLAGS -Werror -Wall -Wextra -Wformat=2 -Wno-type-limits -Wno-unknown-pragmas -Wno-unused-value CACHE INTERNAL "")
 set(MSVC_WARNING_FLAGS /WX /W4 /sdl CACHE INTERNAL "")
 
@@ -335,9 +332,6 @@ if(QUIC_TLS STREQUAL "mitls")
     add_subdirectory(submodules/everest/msquic/msvc/quiccrypto)
 endif()
 
-# CLOG target
-add_subdirectory(submodules/clog/inc)
-
 # Product code
 add_subdirectory(src/core)
 add_subdirectory(src/platform)

Dockerfile

@@ -11,6 +11,7 @@ FROM    source as build
 WORKDIR /src/Debug
 RUN     chmod +x /src/scripts/install-powershell-docker.sh
 RUN     /src/scripts/install-powershell-docker.sh
+ENV     PATH="/root/.dotnet/tools:${PATH}"
 RUN     cmake -DQUIC_ENABLE_LOGGING=OFF -DQUIC_BUILD_TEST=OFF ..
 RUN     cmake --build .
 RUN     openssl ecparam -out server.eckey -noout -name prime256v1 -genkey
@@ -25,7 +26,7 @@ RUN     apt-get update -y \
             libatomic1 \
             && apt-get clean
 COPY    --from=build /src/Debug/bin/Release /bin
-COPY    --from=build /src/Debug/bin/Release/*.so /lib/x86_64-linux-gnu
+COPY    --from=build /src/Debug/bin/Release/*.so /lib/x86_64-linux-gnu/
 COPY    --from=source /src/scripts/run_endpoint.sh /run_endpoint.sh
 COPY    --from=build /src/Debug/server.* /
 RUN     chmod +x /run_endpoint.sh

scripts/build.ps1

@@ -261,12 +261,6 @@ function CMake-Build {
 
     CMake-Execute $Arguments
 
-    # Copy clog to a common location.
-    $ClogPath = Join-Path $BaseArtifactsDir "bin/clog"
-    if (!(Test-Path $ClogPath)) {
-        Copy-Item (Join-Path $BuildDir "submodules/clog") -Destination $ClogPath -Recurse
-    }
-
     if ($IsWindows) {
         Copy-Item (Join-Path $BuildDir "obj" $Config "msquic.lib") $ArtifactsDir
         if (!$DisableTools) {
@@ -298,13 +292,6 @@ function CMake-Build {
 #                     Main Execution                         #
 ##############################################################
 
-if (!$IsWindows) {
-    # Set Linux env variables to include dotnet.
-    $env:PATH+=":$HOME/.dotnet"
-    $env:PATH+=":$HOME/.dotnet/tools"
-    $env:DOTNET_ROOT="$HOME/.dotnet/"
-}
-
 # Generate the build files.
 Log "Generating files..."
 CMake-Generate

scripts/generate_kernel_clog.bat

@@ -11,34 +11,27 @@ echo CMAKE_PROJECTDIR=%CMAKE_PROJECTDIR%
 echo SCOPE_PREFIX=%SCOPE_PREFIX%
 
 set CMAKE_CLOG_OUTPUT_DIRECTORY=%QUIC_BUILD_DIR%\inc
-set CMAKE_CLOG_BINS_DIRECTORY=%QUIC_BUILD_DIR%\submodules\clog
 set CMAKE_CLOG_SIDECAR_DIRECTORY=%CMAKE_SOURCE_DIR%\src\manifest
 set CLOG_SOURCE_DIRECTORY=%CMAKE_SOURCE_DIR%\submodules\clog
 set CLOG_INCLUDE_DIRECTORY=%CMAKE_SOURCE_DIR%\inc
 set CMAKE_CLOG_CONFIG_FILE=%CMAKE_SOURCE_DIR%\src\manifest\msquic.clog_config
 
 echo EnvVars] --------------------------------------------------------------
-set 
+set
 echo -----------------------------------------------------------------------
 
 echo %CMAKE_SOURCE_DIR%
 
 echo Clearing the LIB environment varaible to avoid conflicting with the needs of dotnet
 set LIB=
 
-if NOT EXIST %CMAKE_CLOG_BINS_DIRECTORY%\clog.exe (
-    echo Building CLOG
-    echo dotnet build %CLOG_SOURCE_DIRECTORY%\clog.sln\clog_coreclr.sln -o %CMAKE_CLOG_BINS_DIRECTORY%
-    dotnet build %CLOG_SOURCE_DIRECTORY%\clog.sln\clog_coreclr.sln -o %CMAKE_CLOG_BINS_DIRECTORY% >  %CMAKE_CLOG_BINS_DIRECTORY%.kernel.build.log
-)
-
 pushd %CMAKE_PROJECTDIR%
 echo CLOG Processing Directory %CMAKE_PROJECTDIR%
 for %%i in (*.c* *.c operation.h stream.h connection.h TestHelpers.h) do (
     echo CLOG Processing %%i
     if EXIST %%i (
         if NOT EXIST %CMAKE_CLOG_OUTPUT_DIRECTORY%\%%i.clog.h (
-            %CMAKE_CLOG_BINS_DIRECTORY%\clog.exe -p windows_kernel --scopePrefix %SCOPE_PREFIX% -c %CMAKE_CLOG_CONFIG_FILE% -s %CMAKE_CLOG_SIDECAR_DIRECTORY%\clog.sidecar -i %%i -o %CMAKE_CLOG_OUTPUT_DIRECTORY%\%%i.clog.h
+            clog.exe -p windows_kernel --scopePrefix %SCOPE_PREFIX% -c %CMAKE_CLOG_CONFIG_FILE% -s %CMAKE_CLOG_SIDECAR_DIRECTORY%\clog.sidecar -i %%i -o %CMAKE_CLOG_OUTPUT_DIRECTORY%\%%i.clog.h
             echo %%i
         )
     )

scripts/install-powershell-docker.sh

@@ -9,3 +9,7 @@ apt-get update
 apt-get install -y dotnet-sdk-3.1
 
 dotnet tool install -g powershell
+wget https://github.com/microsoft/CLOG/releases/download/v0.1.1/Microsoft.Logging.CLOG.0.1.1.nupkg
+mkdir nuget
+mv Microsoft.Logging.CLOG.0.1.1.nupkg nuget/Microsoft.Logging.CLOG.0.1.1.nupkg
+dotnet tool install --global --add-source nuget Microsoft.Logging.CLOG

scripts/install-powershell.sh

@@ -1,15 +1,7 @@
 #!/bin/sh
 
-wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh
-chmod +x ./dotnet-install.sh
-./dotnet-install.sh -c Current
-
-export PATH="$PATH:$HOME/.dotnet"
-export PATH="$PATH:$HOME/.dotnet/tools"
-export DOTNET_ROOT="$HOME/.dotnet/"
-
 wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb
 sudo dpkg -i packages-microsoft-prod.deb
 sudo apt-get update
 sudo add-apt-repository universe
-sudo apt-get install -y powershell
+sudo apt-get install -y powershell apt-transport-https

scripts/prepare-machine.ps1

@@ -26,6 +26,34 @@ param (
 Set-StrictMode -Version 'Latest'
 $PSDefaultParameterValues['*:ErrorAction'] = 'Stop'
 
+# Root directory of the project.
+$RootDir = Split-Path $PSScriptRoot -Parent
+$NuGetPath = Join-Path $RootDir "nuget"
+
+function Install-ClogTool {
+    param($NuGetName, $ToolName, $DownloadUrl)
+    New-Item -Path $NuGetPath -ItemType Directory -Force | Out-Null
+    $NuGetFile = Join-Path $NuGetPath $NuGetName
+    $OldProgressPreference = $ProgressPreference
+    $ProgressPreference = 'SilentlyContinue'
+    try {
+        Invoke-WebRequest -Uri "$DownloadUrl/$NuGetName" -OutFile $NuGetFile
+        dotnet tool install --global --add-source $NuGetPath $ToolName
+    } catch {
+        Write-Warning "Clog could not be installed. Building with logs will not work"
+        Write-Warning $_
+    } finally {
+        $ProgressPreference = $OldProgressPreference
+    }
+}
+
+if (($Configuration -eq "Dev") -or ($Configuration -eq "Build")) {
+        $NuGetName = "Microsoft.Logging.CLOG.0.1.2.nupkg"
+        $ToolName = "Microsoft.Logging.CLOG"
+        $DownloadUrl = "https://github.com/microsoft/CLOG/releases/download/v0.1.2"
+        Install-ClogTool -NuGetName $NuGetName -ToolName $ToolName -DownloadUrl $DownloadUrl
+}
+
 if ($IsWindows) {
 
     if ($Configuration -eq "Dev") {
@@ -40,6 +68,12 @@ if ($IsWindows) {
         reg.exe add $TlsClientKeyPath /v DisabledByDefault /t REG_DWORD /d 0 /f | Out-Null
         reg.exe add $TlsClientKeyPath /v Enabled /t REG_DWORD /d 1 /f | Out-Null
     }
+    if ($Configuration -eq "Test") {
+        $NuGetName = "Microsoft.Logging.CLOG2Text.Windows.0.1.2.nupkg"
+        $ToolName = "Microsoft.Logging.CLOG2Text.Windows"
+        $DownloadUrl = "https://github.com/microsoft/CLOG/releases/download/v0.1.2"
+        Install-ClogTool -NuGetName $NuGetName -ToolName $ToolName -DownloadUrl $DownloadUrl
+    }
 
 } else {
     switch ($Configuration) {
@@ -65,6 +99,11 @@ if ($IsWindows) {
             Write-Host "[$(Get-Date)] Setting core dump pattern..."
             sudo sh -c "echo -n '%e.%p.%t.core' > /proc/sys/kernel/core_pattern"
             #sudo cat /proc/sys/kernel/core_pattern
+
+            $NuGetName = "Microsoft.Logging.CLOG2Text.Lttng.0.1.2.nupkg"
+            $ToolName = "Microsoft.Logging.CLOG2Text.Lttng"
+            $DownloadUrl = "https://github.com/microsoft/CLOG/releases/download/v0.1.2"
+            Install-ClogTool -NuGetName $NuGetName -ToolName $ToolName -DownloadUrl $DownloadUrl
         }
         "Dev" {
             sudo apt-add-repository ppa:lttng/stable-2.10

src/bin/CMakeLists.txt

@@ -44,9 +44,9 @@ set(PUBLIC_HEADERS
     ../inc/msquic_linux.h
     ../inc/quic_sal_stub.h)
 
-install(TARGETS msquic EXPORT msquic DESTINATION "${main_lib_dest}")
-install(FILES ${PUBLIC_HEADERS} DESTINATION "${include_dest}")
+#install(TARGETS msquic EXPORT msquic DESTINATION "${main_lib_dest}")
+#install(FILES ${PUBLIC_HEADERS} DESTINATION "${include_dest}")
 
-configure_file(msquic-config.cmake.in ${CMAKE_BINARY_DIR}/msquic-config.cmake )
-install(FILES ${CMAKE_BINARY_DIR}/msquic-config.cmake DESTINATION ${msquic_dest})
-install(EXPORT msquic DESTINATION ${msquic_dest})
+#configure_file(msquic-config.cmake.in ${CMAKE_BINARY_DIR}/msquic-config.cmake )
+#install(FILES ${CMAKE_BINARY_DIR}/msquic-config.cmake DESTINATION ${msquic_dest})
+#install(EXPORT msquic DESTINATION ${msquic_dest})

src/core/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 set(SOURCES
     ack_tracker.c

src/core/unittest/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 include_directories(${PROJECT_SOURCE_DIR}/src/core)
 include_directories(${PROJECT_SOURCE_DIR}/submodules/googletest/googletest/include)

src/manifest/create_kernel_clog_dependencies.bat

@@ -6,6 +6,8 @@ set QUIC_BUILD_DIR=%SOLUTION_DIR%\build\winkernel\%PLATFORM_ARG%_%CONFIGURATION_
 mkdir %QUIC_BUILD_DIR%\inc
 mc.exe -um -h %QUIC_BUILD_DIR%\inc -r %QUIC_BUILD_DIR%\inc %SOLUTION_DIR%\src\manifest\MsQuicEtw.man
 
+clog --installDependencies %SOLUTION_DIR%\build\clog
+
 cmd /c %SOLUTION_DIR%\scripts\generate_kernel_clog.bat %QUIC_BUILD_DIR% %SOLUTION_DIR% %SOLUTION_DIR%\src\core CORE
 cmd /c %SOLUTION_DIR%\scripts\generate_kernel_clog.bat %QUIC_BUILD_DIR% %SOLUTION_DIR% %SOLUTION_DIR%\src\platform PLATFORM
 cmd /c %SOLUTION_DIR%\scripts\generate_kernel_clog.bat %QUIC_BUILD_DIR% %SOLUTION_DIR% %SOLUTION_DIR%\src\bin\winkernel BIN

src/manifest/manifest.kernel.vcxproj

@@ -70,7 +70,7 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <ClCompile>
-      <AdditionalIncludeDirectories>..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;..\..\submodules\clog\inc;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>..\inc;$(SolutionDir)build\winkernel\$(Platform)_$(Configuration)_schannel\inc;$(SolutionDir)build\clog;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <PreBuildEvent>
       <Command>cmd /C $(SolutionDir)\src\manifest\create_kernel_clog_dependencies.bat $(SolutionDir) $(Platform) $(Configuration)</Command>

src/manifest/msquic.clog_config

@@ -77,6 +77,6 @@
   "SourceCodeMacros": [
   ],
   "ChainedConfigFiles": [
-    "../../submodules/clog/inc/defaults.clog_config"
+    "DEFAULTS"
   ]
 }

src/platform/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
     set(SOURCES

src/platform/unittest/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 include_directories(${PROJECT_SOURCE_DIR}/src/platform)
 include_directories(${PROJECT_SOURCE_DIR}/submodules/googletest/googletest/include)

src/test/bin/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 include_directories(${PROJECT_SOURCE_DIR}/src/test)
 include_directories(${PROJECT_SOURCE_DIR}/submodules/googletest/googletest/include)

src/test/lib/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Licensed under the MIT License.
 
 # Include for CLOG processing
-include("${CLOG_INCLUDE_DIRECTORY}/CLog.make")
+include(${CMAKE_CLOG_GENERATE_FILE})
 
 set(SOURCES
     ApiTest.cpp