Adding the support of RFP=1 for Linux Consumption (Azure#8374)

* lin con rfp=1 * minor change * fixed an issue * logs for testings * test without metrics * change in getpackage type * cleanup. * Unit test fix. * Added unit tests * Removed the code witch is not needed. * Fix after testing. * Addressed the review comments. * Passing FileSystem object. * Added unit tests * typos. * Addressed the review comments. * Feedback * rfp=1 is now Squashfs * Triggering build which was canceled because of no agent. * Removed the test which is not needed because of the recent change.
slsec · Aug 3, 2022 · 1a62cef · 1a62cef
1 parent 172f833
commit 1a62cef
Show file tree

Hide file tree

Showing 12 changed files with 365 additions and 55 deletions.
diff --git a/src/WebJobs.Script.WebHost/Management/InstanceManager.cs b/src/WebJobs.Script.WebHost/Management/InstanceManager.cs
@@ -314,7 +314,7 @@ private async Task ApplyContext(HostAssignmentContext assignmentContext)
             var options = _optionsFactory.Create(ScriptApplicationHostOptionsSetup.SkipPlaceholder);
             RunFromPackageContext pkgContext = assignmentContext.GetRunFromPkgContext();
 
-            if (_environment.SupportsAzureFileShareMount())
+            if (_environment.SupportsAzureFileShareMount() || pkgContext.IsRunFromLocalPackage())
             {
                 var azureFilesMounted = false;
                 if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
@@ -334,13 +334,30 @@ private async Task ApplyContext(HostAssignmentContext assignmentContext)
                         _logger.LogWarning("App is configured to use both Run-From-Package and AzureFiles. Run-From-Package will take precedence");
                     }
                     var blobContextApplied =
-                        await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath,
+                        await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath,
                             azureFilesMounted, false);
 
                     if (!blobContextApplied && azureFilesMounted)
                     {
-                        _logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyBlobPackageContext)}. Attempting to use local disk instead");
-                        await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
+                        _logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyRunFromPackageContext)}. Attempting to use local disk instead");
+                        await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, false);
+                    }
+                }
+                else if (pkgContext.IsRunFromLocalPackage())
+                {
+                    if (!azureFilesMounted)
+                    {
+                        const string mountErrorMessage = "App Run-From-Package is set as '1'. AzureFiles is needed but is not configured.";
+                        _logger.LogWarning(mountErrorMessage);
+                        throw new Exception(mountErrorMessage);
+                    }
+
+                    var blobContextApplied =
+                        await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, azureFilesMounted);
+
+                    if (!blobContextApplied)
+                    {
+                        _logger.LogWarning($"Failed to {nameof(_runFromPackageHandler.ApplyRunFromPackageContext)}.");
                     }
                 }
                 else
@@ -352,7 +369,7 @@ await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptP
             {
                 if (pkgContext.IsRunFromPackage(options, _logger))
                 {
-                    await _runFromPackageHandler.ApplyBlobPackageContext(pkgContext, options.ScriptPath, false);
+                    await _runFromPackageHandler.ApplyRunFromPackageContext(pkgContext, options.ScriptPath, false);
                 }
                 else if (assignmentContext.IsAzureFilesContentShareConfigured(_logger))
                 {

diff --git a/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IPackageDownloadHandler.cs b/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IPackageDownloadHandler.cs
@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using System.IO.Abstractions;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 

diff --git a/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IRunFromPackageHandler.cs b/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/IRunFromPackageHandler.cs
@@ -10,7 +10,7 @@ public interface IRunFromPackageHandler
     {
         Task<bool> MountAzureFileShare(HostAssignmentContext assignmentContext);
 
-        Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted,
+        Task<bool> ApplyRunFromPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted,
             bool throwOnFailure = true);
     }
 }
diff --git a/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs b/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/PackageDownloadHandler.cs
@@ -3,11 +3,13 @@
 
 using System;
 using System.IO;
+using System.IO.Abstractions;
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
 using System.Threading;
 using System.Threading.Tasks;
+using DryIoc;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;
 using Microsoft.Extensions.Logging;
@@ -25,20 +27,34 @@ public class PackageDownloadHandler : IPackageDownloadHandler
         private readonly IBashCommandHandler _bashCommandHandler;
         private readonly ILogger<PackageDownloadHandler> _logger;
         private readonly IMetricsLogger _metricsLogger;
+        private readonly IEnvironment _environment;
+        private readonly IFileSystem _fileSystem;
 
         public PackageDownloadHandler(IHttpClientFactory httpClientFactory, IManagedIdentityTokenProvider managedIdentityTokenProvider,
-            IBashCommandHandler bashCommandHandler, ILogger<PackageDownloadHandler> logger,
+            IBashCommandHandler bashCommandHandler, IEnvironment environment, IFileSystem fileSystem, ILogger<PackageDownloadHandler> logger,
             IMetricsLogger metricsLogger)
         {
             _httpClient = httpClientFactory?.CreateClient() ?? throw new ArgumentNullException(nameof(httpClientFactory));
             _managedIdentityTokenProvider = managedIdentityTokenProvider ?? throw new ArgumentNullException(nameof(managedIdentityTokenProvider));
             _bashCommandHandler = bashCommandHandler ?? throw new ArgumentNullException(nameof(bashCommandHandler));
+            _environment = environment ?? throw new ArgumentNullException(nameof(environment));
+            _fileSystem = fileSystem ?? FileUtility.Instance;
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
         }
 
+        /// <summary>
+        /// Download the package from blob storage or fileshare.
+        /// </summary>
+        /// <param name="pkgContext">Package Context.</param>
+        /// <returns>Path of the downloaded package.</returns>
         public async Task<string> Download(RunFromPackageContext pkgContext)
         {
+            if (pkgContext.IsRunFromLocalPackage())
+            {
+                 return CopyPackageFile(pkgContext);
+            }
+
             if (Utility.TryCleanUrl(pkgContext.Url, out var cleanedUrl))
             {
                 _logger.LogDebug("Downloading app contents from '{cleanedUrl}'", cleanedUrl);
@@ -214,5 +230,59 @@ private async Task<bool> IsResourceAccessibleWithoutAuthorization(string resourc
                 return false;
             }
         }
+
+        private string CopyPackageFile(RunFromPackageContext pkgContext)
+        {
+            var packageFolderPath = _environment.GetSitePackagesPath();
+            Action<string> copyPackageFileFailed = (message) =>
+            {
+                _logger.LogWarning(message);
+                throw new InvalidOperationException(message);
+            };
+
+            if (!_fileSystem.Directory.Exists(packageFolderPath))
+            {
+                copyPackageFileFailed($"{ScriptConstants.SitePackagesFolderName} folder doesn't exist at {packageFolderPath}.");
+            }
+
+            var packageNameTxtPath = _environment.GetSitePackageNameTxtPath();
+            if (!_fileSystem.File.Exists(packageNameTxtPath))
+            {
+                copyPackageFileFailed($"{ScriptConstants.SitePackageNameTxtFileName} doesn't exist at {packageNameTxtPath}.");
+            }
+
+            var packageFileName = _fileSystem.File.ReadAllText(packageNameTxtPath);
+
+            if (string.IsNullOrEmpty(packageFileName))
+            {
+                copyPackageFileFailed($"{ScriptConstants.SitePackageNameTxtFileName} is empty at {packageNameTxtPath}.");
+            }
+
+            var packageFilePath = _fileSystem.Path.Combine(packageFolderPath, packageFileName);
+            if (!_fileSystem.File.Exists(packageFilePath))
+            {
+                copyPackageFileFailed($"{packageFileName} doesn't exist at {packageFilePath}.");
+            }
+
+            var tmpPath = _fileSystem.Path.GetTempPath();
+            var fileName = _fileSystem.Path.GetFileName(packageFileName);
+            var filePath = _fileSystem.Path.Combine(tmpPath, fileName);
+
+            var copyMetricName = pkgContext.IsWarmUpRequest
+                    ? MetricEventNames.LinuxContainerSpecializationZipMountCopyWarmup
+                    : MetricEventNames.LinuxContainerSpecializationZipMountCopy;
+
+            using (_metricsLogger.LatencyEvent(copyMetricName))
+            {
+                _fileSystem.File.Copy(packageFilePath, filePath, true);
+
+                var fileInfo = _fileSystem.FileInfo.FromFileName(filePath);
+                _logger.LogInformation($"Downloaded Package size is {fileInfo.Length}");
+            }
+
+            _logger.LogInformation($"{nameof(CopyPackageFile)} was successful. {packageFileName} was copied from {packageFilePath} to {filePath}.");
+
+            return filePath;
+        }
     }
 }
diff --git a/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/RunFromPackageHandler.cs b/src/WebJobs.Script.WebHost/Management/LinuxSpecialization/RunFromPackageHandler.cs
@@ -38,7 +38,7 @@ public RunFromPackageHandler(IEnvironment environment, IMeshServiceClient meshSe
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted, bool throwOnFailure = true)
+        public async Task<bool> ApplyRunFromPackageContext(RunFromPackageContext pkgContext, string targetPath, bool azureFilesMounted, bool throwOnFailure = true)
         {
             try
             {
@@ -51,8 +51,10 @@ public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext
                         EnvironmentSettingNames.DefaultLocalSitePackagesPath)
                     : string.Empty;
 
-                // download zip and extract
-                var filePath = await _packageDownloadHandler.Download(pkgContext);
+                // download zip
+                string filePath = await _packageDownloadHandler.Download(pkgContext);
+
+                // extract zip
                 await UnpackPackage(filePath, targetPath, pkgContext, localSitePackagesPath);
 
                 string bundlePath = Path.Combine(targetPath, "worker-bundle");
@@ -65,7 +67,7 @@ public async Task<bool> ApplyBlobPackageContext(RunFromPackageContext pkgContext
             }
             catch (Exception e)
             {
-                _logger.LogDebug(e, nameof(ApplyBlobPackageContext));
+                _logger.LogDebug(e, nameof(ApplyRunFromPackageContext));
                 if (throwOnFailure)
                 {
                     throw;
@@ -128,8 +130,8 @@ private async Task UnpackPackage(string filePath, string scriptPath, RunFromPack
 
         private CodePackageType GetPackageType(string filePath, RunFromPackageContext pkgContext)
         {
-            // cloud build always builds squashfs
-            if (pkgContext.IsScmRunFromPackage())
+            // cloud build always builds squashfs.
+            if (pkgContext.IsScmRunFromPackage() || pkgContext.IsRunFromLocalPackage())
             {
                 return CodePackageType.Squashfs;
             }

diff --git a/src/WebJobs.Script.WebHost/Models/RunFromPackageContext.cs b/src/WebJobs.Script.WebHost/Models/RunFromPackageContext.cs
@@ -29,11 +29,22 @@ public bool IsScmRunFromPackage()
             return string.Equals(EnvironmentVariableName, EnvironmentSettingNames.ScmRunFromPackage, StringComparison.OrdinalIgnoreCase);
         }
 
+        public bool IsWebsiteRunFromPackage()
+        {
+            return string.Equals(EnvironmentVariableName, EnvironmentSettingNames.AzureWebsiteRunFromPackage, StringComparison.OrdinalIgnoreCase) ||
+                string.Equals(EnvironmentVariableName, EnvironmentSettingNames.AzureWebsiteAltZipDeployment, StringComparison.OrdinalIgnoreCase);
+        }
+
         public bool IsRunFromPackage(ScriptApplicationHostOptions options, ILogger logger)
         {
             return (IsScmRunFromPackage() && ScmRunFromPackageBlobExists(options, logger)) || (!IsScmRunFromPackage() && !string.IsNullOrEmpty(Url) && Url != "1");
         }
 
+        public bool IsRunFromLocalPackage()
+        {
+            return IsWebsiteRunFromPackage() && string.Equals(Url, "1", StringComparison.OrdinalIgnoreCase);
+        }
+
         private bool ScmRunFromPackageBlobExists(ScriptApplicationHostOptions options, ILogger logger)
         {
             var blobExists = options.IsScmRunFromPackage;

diff --git a/src/WebJobs.Script/Diagnostics/MetricEventNames.cs b/src/WebJobs.Script/Diagnostics/MetricEventNames.cs
@@ -79,6 +79,8 @@ public static class MetricEventNames
         public const string LinuxContainerSpecializationZipDownloadWarmup = "linux.container.specialization.zip.download.warmup";
         public const string LinuxContainerSpecializationZipWrite = "linux.container.specialization.zip.write";
         public const string LinuxContainerSpecializationZipWriteWarmup = "linux.container.specialization.zip.write.warmup";
+        public const string LinuxContainerSpecializationZipMountCopy = "linux.container.specialization.zip.mountcopy";
+        public const string LinuxContainerSpecializationZipMountCopyWarmup = "linux.container.specialization.zip.mountcopy.warmup";
         public const string LinuxContainerSpecializationZipHead = "linux.container.specialization.zip.head";
         public const string LinuxContainerSpecializationZipHeadWarmup = "linux.container.specialization.zip.head.warmup";
         public const string LinuxContainerSpecializationFuseMount = "linux.container.specialization.mount";

diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs
@@ -104,6 +104,21 @@ public static bool AzureFilesAppSettingsExist(this IEnvironment environment)
                    !string.IsNullOrEmpty(environment.GetEnvironmentVariable(AzureFilesContentShare));
         }
 
+        public static string GetAzureWebsiteHomePath(this IEnvironment environment)
+        {
+            return environment.GetEnvironmentVariable(AzureWebsiteHomePath);
+        }
+
+        public static string GetSitePackagesPath(this IEnvironment environment)
+        {
+            return Path.Combine(environment.GetAzureWebsiteHomePath(), ScriptConstants.DataFolderName, ScriptConstants.SitePackagesFolderName);
+        }
+
+        public static string GetSitePackageNameTxtPath(this IEnvironment environment)
+        {
+            return Path.Combine(environment.GetSitePackagesPath(), ScriptConstants.SitePackageNameTxtFileName);
+        }
+
         public static bool IsCoreTools(this IEnvironment environment)
         {
             return !string.IsNullOrEmpty(environment.GetEnvironmentVariable(CoreToolsEnvironment));

diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs
@@ -187,6 +187,9 @@ public static class ScriptConstants
         public const string HelpLinkKey = "MS_HelpLink";
         public const string ErrorCodeKey = "MS_ErrorCode";
 
+        public const string DataFolderName = "data";
+        public const string SitePackagesFolderName = "SitePackages";
+        public const string SitePackageNameTxtFileName = "packagename.txt";
         // Diagnostic sources
         public const string HostDiagnosticSourcePrefix = "Microsoft.Azure.Functions.Host.";
         public const string HostDiagnosticSourceDebugEventNamePrefix = "debug-";