Merge pull request fsprojects#1052 from colinbull/master

Support for authentication and complex hosts.

docs/content/http-dependencies.md

@@ -22,3 +22,21 @@ If you want to reference the file in one of your project files then add an entry
 
 This will reference the linked file directly into your project.
 By default the linked file will be visible under ``paket-files`` folder in project.
+
+## Options on http dependencies
+
+When referencing a file using a http dependency, there are several options that help you to deal with things like authentication and file name. 
+The pattern expected is 
+
+	http url [FileSpec] [SourceName]
+
+* **FileSpec** - This allows you to define the path to which the file that is downloaded will be written to. For example specfying the following
+
+		http http://www.fssnip.net/raw/1M/test1.fs src/test1.fs
+
+	will write the file to `paket-files\www.fssnip.net\src\test.fs` 
+
+* **SourceName** - The source name allows you to override the folder which the downloaded file is written to and also acts as a key to lookup any authentication 
+that maybe assoicated for that key. For example if I had added an authentication source using [``paket config add-authentication MySource``](commands\config.html)
+then each time paket extracts a HTTP dependency with `MySource` as a `SourceName` the credentails will be made part of the HTTP request. If no keys exist in the authentication store
+then the request will be made without any authentication headers.

src/Paket.Core/ConfigFile.fs

@@ -122,22 +122,27 @@ let getSourceNodes (credentialsNode : XmlNode) (source) =
     |> Seq.filter (fun n -> n.Attributes.["source"].Value = source)
     |> Seq.toList
 
-
-/// Get the credential from the credential store for a specific source
-let GetCredentials (source : string) =
+/// Get the credential from the credential store for a specific source and valdiates against the url
+let GetCredentialsForUrl (source : string) url =
     let credentialsNode = getConfigNode "credentials" |> returnOrFail
 
     match getSourceNodes credentialsNode source with
     | sourceNode::_ ->
         let username,password = getAuthFromNode sourceNode
         let auth = {Username = username; Password = password}
-        if checkCredentials(source, Some(auth)) then
+        if checkCredentials(url, Some(auth)) then
             Some(username,password)
         else
             traceWarnfn "credentials for %s source are invalid" source  
             None
     | [] -> None
 
+/// Get the credential from the credential store for a specific source
+let GetCredentials (source : string) =
+    GetCredentialsForUrl source source
+
+
+
 let AddCredentials (source, username, password) = trial {
         let! credentialsNode = getConfigNode "credentials"
 

src/Paket.Core/DependenciesFile.fs

@@ -113,7 +113,11 @@ module DependenciesFileParser =
 
     let private ``parse http source`` trimmed =
         let parts = parseDependencyLine trimmed
-        let getParts (projectSpec:string) fileSpec =
+
+        let removeInvalidChars (str:string) = 
+            System.Text.RegularExpressions.Regex.Replace(str, "[:@\,]", "_")
+
+        let getParts (projectSpec:string) fileSpec projectName =
             let projectSpec = projectSpec.TrimEnd('/')
             let ``project spec``, commit =
                 match projectSpec.IndexOf('/', 8) with // 8 = "https://".Length
@@ -129,12 +133,14 @@ module DependenciesFileParser =
             let owner =
                 match ``project spec``.IndexOf("://") with
                 | -1 -> ``project spec``
-                | pos ->  ``project spec``.Substring(pos+3)
-            HttpLink(``project spec``), (owner, "", Some commit), fileName
-        match parts with
-        | [| _; projectSpec; |] -> getParts projectSpec String.Empty
-        | [| _; projectSpec; fileSpec |] -> getParts projectSpec fileSpec
-        | _ -> failwithf "invalid http-reference specification:%s     %s" Environment.NewLine trimmed
+                | pos ->  ``project spec``.Substring(pos+3) |> removeInvalidChars
+            HttpLink(``project spec``), (owner, projectName, Some commit), fileName
+
+        match parseDependencyLine trimmed with
+        | [|spec; url |] -> getParts url "" ""
+        | [|spec; url; fileSpec |] -> getParts url fileSpec ""
+        | [|spec; url; fileSpec; projectName |] -> getParts url fileSpec projectName
+        | _ ->  failwithf "invalid http-reference specification:%s     %s" Environment.NewLine trimmed
 
     type private ParserOption =
     | ReferencesMode of bool

src/Paket.Core/ModuleResolver.fs

@@ -8,7 +8,7 @@ open Paket.Requirements
 
 type SingleSourceFileOrigin = 
 | GitHubLink 
-| GistLink 
+| GistLink
 | HttpLink of string
 
 // Represents details on a dependent source file.
@@ -22,7 +22,7 @@ type UnresolvedSourceFile =
     override this.ToString() =
         let name = if this.Name = Constants.FullProjectSourceFileName then "" else " " + this.Name
         match this.Origin with
-        | HttpLink url -> sprintf "http %s%s %s" url this.Commit.Value this.Name
+        | HttpLink url -> sprintf "http %s%s %s" url (defaultArg this.Commit "") this.Name
         | _ ->
             let link = 
                 match this.Origin with

src/Paket.Core/PackageSources.fs

@@ -66,7 +66,7 @@ let private parseAuth(text, source) =
 
         let basicAuth = toBasicAuth auth
         if(basicAuth.Username = "" && basicAuth.Password = "") then
-            ConfigFile.GetCredentials source
+            ConfigFile.GetCredentials source 
             |> Option.map (fun (username,password) -> 
                             ConfigAuthentication(username, password))
         else

src/Paket.Core/RemoteDownload.fs

@@ -37,14 +37,20 @@ let downloadDependenciesFile(rootPath,parserF,remoteFile:ModuleResolver.Resolved
 
     let dependenciesFileName = remoteFile.Name.Replace(fi.Name,Constants.DependenciesFileName)
 
-    let url = 
+    let auth, url = 
         match remoteFile.Origin with
         | ModuleResolver.GitHubLink -> 
-            rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName
+            None, rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit dependenciesFileName
         | ModuleResolver.GistLink -> 
-            rawGistFileUrl remoteFile.Owner remoteFile.Project dependenciesFileName
-        | ModuleResolver.HttpLink url -> url.Replace(remoteFile.Name,Constants.DependenciesFileName)
-    let! result = safeGetFromUrl(None,url,null)
+            None, rawGistFileUrl remoteFile.Owner remoteFile.Project dependenciesFileName
+        | ModuleResolver.HttpLink url -> 
+            let url = url.Replace(remoteFile.Name,Constants.DependenciesFileName)
+            let auth = 
+                ConfigFile.GetCredentialsForUrl remoteFile.Project url
+                |> Option.map (fun (un, pwd) -> { Username = un; Password = pwd })
+            auth, url
+
+    let! result = safeGetFromUrl(auth,url,null)
 
     match result with
     | Some text when parserF text ->        
@@ -117,7 +123,10 @@ let downloadRemoteFiles(remoteFile:ResolvedSourceFile,destination) = async {
         return! downloadFromUrl(None,rawFileUrl remoteFile.Owner remoteFile.Project remoteFile.Commit remoteFile.Name) destination
     | SingleSourceFileOrigin.HttpLink(origin), _ ->
         let url = origin + remoteFile.Commit
-        do! downloadFromUrl(None, url) destination
+        let auth =
+            ConfigFile.GetCredentialsForUrl remoteFile.Project url
+            |> Option.map (fun (un, pwd) -> { Username = un; Password = pwd })
+        do! downloadFromUrl(auth, url) destination
         match Path.GetExtension(destination).ToLowerInvariant() with
         | ".zip" ->
             let targetFolder = FileInfo(destination).Directory.FullName

src/Paket/Paket.fsproj

@@ -37,7 +37,7 @@
     <StartArguments>install</StartArguments>
     <StartAction>Project</StartAction>
     <StartProgram>paket.exe</StartProgram>
-    <StartWorkingDirectory>d:\code\RegexProvider</StartWorkingDirectory>
+    <StartWorkingDirectory>C:\Appdev\petroineos.var</StartWorkingDirectory>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <DebugType>pdbonly</DebugType>

tests/Paket.Tests/DependenciesFile/ParserSpecs.fs

@@ -343,6 +343,39 @@ let ``should read http source file from config without quotes with file specs``(
             Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net"
             Commit = Some "/raw/1M/1" } ]
 
+
+[<Test>]
+let ``should read http source file from config without quotes with file specs and project and query string after filename``() =
+    let config = """http http://server-stash:7658/projects/proj1/repos/repo1/browse/Source/SolutionFolder/Rabbit.fs?at=a5457f3d811830059cd39d583f264eab340c273d&raw Rabbit.fs project"""
+    let dependencies = DependenciesFile.FromCode(config)
+    dependencies.RemoteFiles
+    |> shouldEqual
+        [ { Owner = "server-stash_7658"
+            Project = "project"
+            Name = "Rabbit.fs"
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://server-stash:7658"
+            Commit = Some "/projects/proj1/repos/repo1/browse/Source/SolutionFolder/Rabbit.fs?at=a5457f3d811830059cd39d583f264eab340c273d&raw" }
+        ]
+
+[<Test>]
+let ``should read http source file from config without quotes with file specs and project``() =
+    let config = """http http://www.fssnip.net/raw/1M test1.fs project
+                    http http://www.fssnip.net/raw/1M/1 src/test2.fs project"""
+    let dependencies = DependenciesFile.FromCode(config)
+    dependencies.RemoteFiles
+    |> shouldEqual
+        [ { Owner = "www.fssnip.net"
+            Project = "project"
+            Name = "test1.fs"
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net"
+            Commit = Some "/raw/1M" }
+          { Owner = "www.fssnip.net"
+            Project = "project"
+            Name = "src/test2.fs"
+            Origin = ModuleResolver.SingleSourceFileOrigin.HttpLink "http://www.fssnip.net"
+            Commit = Some "/raw/1M/1" } ]
+
+
 [<Test>]
 let ``should read gist source file from config without quotes with file specs``() =
     let config = """gist Thorium/1972308 gistfile1.fs