Merge pull request #76 from schnusch/unstable

update unstable packages from git to latest branch

nix_update/__init__.py

@@ -95,7 +95,7 @@ def git_has_diff(git_dir: str, package: Package) -> bool:
 
 
 def format_commit_message(package: Package) -> str:
-    new_version = package.new_version
+    new_version = getattr(package.new_version, "number", None)
     if (
         new_version
         and package.old_version != new_version
@@ -112,7 +112,7 @@ def git_commit(git_dir: str, package: Package) -> None:
     msg = format_commit_message(package)
     new_version = package.new_version
     run(["git", "-C", git_dir, "add", package.filename], stdout=None)
-    if new_version and package.old_version != new_version:
+    if new_version and package.old_version != new_version.number:
         run(
             ["git", "-C", git_dir, "commit", "--verbose", "--message", msg], stdout=None
         )

nix_update/eval.py

@@ -5,7 +5,7 @@
 from .errors import UpdateError
 from .options import Options
 from .utils import run
-from .version.version import VersionPreference
+from .version.version import Version, VersionPreference
 
 
 @dataclass
@@ -35,7 +35,7 @@ class Package:
 
     raw_version_position: InitVar[Optional[Dict[str, Any]]]
 
-    new_version: Optional[str] = None
+    new_version: Optional[Version] = None
     version_position: Optional[Position] = field(init=False)
 
     def __post_init__(self, raw_version_position: Optional[Dict[str, Any]]) -> None:
@@ -61,7 +61,7 @@ def eval_expression(import_path: str, attr: str) -> str:
         builtins.unsafeGetAttrPos "src" pkg;
     in {{
       name = pkg.name;
-      old_version = (builtins.parseDrvName pkg.name).version;
+      old_version = pkg.version or (builtins.parseDrvName pkg.name).version;
       inherit raw_version_position;
       filename = position.file;
       line = position.line;

nix_update/update.py

@@ -9,20 +9,22 @@
 from .options import Options
 from .utils import info, run
 from .version import fetch_latest_version
-from .version.version import VersionPreference
+from .version.version import Version, VersionPreference
 
 
 def replace_version(package: Package) -> bool:
+    assert package.new_version is not None
     old_version = package.old_version
-    new_version = package.new_version
-    assert new_version is not None
+    new_version = package.new_version.number
     if new_version.startswith("v"):
         new_version = new_version[1:]
 
     if old_version != new_version:
         info(f"Update {old_version} -> {new_version} in {package.filename}")
         with fileinput.FileInput(package.filename, inplace=True) as f:
             for line in f:
+                if package.new_version.rev:
+                    line = line.replace(package.rev, package.new_version.rev)
                 print(line.replace(old_version, new_version), end="")
     else:
         info(f"Not updating version, already {old_version}")
@@ -131,7 +133,7 @@ def update_version(
     package: Package, version: str, preference: VersionPreference, version_regex: str
 ) -> bool:
     if preference == VersionPreference.FIXED:
-        new_version = version
+        new_version = Version(version)
     else:
         if not package.url:
             if package.urls:
@@ -140,12 +142,23 @@ def update_version(
                 raise UpdateError(
                     "Could not find a url in the derivations src attribute"
                 )
-        new_version = fetch_latest_version(package.url, preference, version_regex)
+        version
+        if preference != VersionPreference.BRANCH:
+            branch = None
+        elif version == "branch":
+            # fallback
+            branch = "HEAD"
+        else:
+            assert version.startswith("branch=")
+            branch = version[7:]
+        new_version = fetch_latest_version(
+            package.url, preference, version_regex, branch
+        )
     package.new_version = new_version
     position = package.version_position
-    if new_version == package.old_version and position:
+    if new_version.number == package.old_version and position:
         recovered_version = old_version_from_git(
-            position.file, position.line, new_version
+            position.file, position.line, new_version.number
         )
         if recovered_version:
             package.old_version = recovered_version

nix_update/version/__init__.py

@@ -1,11 +1,12 @@
 import re
+from functools import partial
 from typing import Callable, List, Optional
 from urllib.parse import ParseResult, urlparse
 
 from ..errors import VersionError
 from .crate import fetch_crate_versions
-from .github import fetch_github_versions
-from .gitlab import fetch_gitlab_versions
+from .github import fetch_github_snapshots, fetch_github_versions
+from .gitlab import fetch_gitlab_snapshots, fetch_gitlab_versions
 from .pypi import fetch_pypi_versions
 from .rubygems import fetch_rubygem_versions
 from .savannah import fetch_savannah_versions
@@ -31,14 +32,19 @@
     fetch_sourcehut_versions,
 ]
 
+branch_snapshots_fetchers: List[Callable[[ParseResult, str], List[Version]]] = [
+    fetch_github_snapshots,
+    fetch_gitlab_snapshots,
+]
+
 
-def extract_version(version: str, version_regex: str) -> Optional[str]:
+def extract_version(version: Version, version_regex: str) -> Optional[Version]:
     pattern = re.compile(version_regex)
-    match = re.match(pattern, version)
+    match = re.match(pattern, version.number)
     if match is not None:
         group = match.group(1)
         if group is not None:
-            return group
+            return Version(group, prerelease=version.prerelease, rev=version.rev)
     return None
 
 
@@ -50,25 +56,31 @@ def is_unstable(version: Version, extracted: str) -> bool:
 
 
 def fetch_latest_version(
-    url_str: str, preference: VersionPreference, version_regex: str
-) -> str:
+    url_str: str,
+    preference: VersionPreference,
+    version_regex: str,
+    branch: Optional[str] = None,
+) -> Version:
     url = urlparse(url_str)
 
     unstable: List[str] = []
     filtered: List[str] = []
-    for fetcher in fetchers:
+    used_fetchers = fetchers
+    if preference == VersionPreference.BRANCH:
+        used_fetchers = [partial(f, branch=branch) for f in branch_snapshots_fetchers]
+    for fetcher in used_fetchers:
         versions = fetcher(url)
         if versions == []:
             continue
         final = []
         for version in versions:
-            extracted = extract_version(version.number, version_regex)
+            extracted = extract_version(version, version_regex)
             if extracted is None:
                 filtered.append(version.number)
             elif preference == VersionPreference.STABLE and is_unstable(
-                version, extracted
+                version, extracted.number
             ):
-                unstable.append(extracted)
+                unstable.append(extracted.number)
             else:
                 final.append(extracted)
         if final != []:

nix_update/version/github.py

@@ -34,3 +34,29 @@ def fetch_github_versions(url: ParseResult) -> List[Version]:
     tree = ET.fromstring(resp.read())
     releases = tree.findall(".//{http://www.w3.org/2005/Atom}entry")
     return [version_from_entry(x) for x in releases]
+
+
+def fetch_github_snapshots(url: ParseResult, branch: str) -> List[Version]:
+    if url.netloc != "github.com":
+        return []
+    parts = url.path.split("/")
+    owner, repo = parts[1], parts[2]
+    repo = re.sub(r"\.git$", "", repo)
+    feed_url = f"https://github.com/{owner}/{repo}/commits/{branch}.atom"
+    info(f"fetch {feed_url}")
+    resp = urllib.request.urlopen(feed_url)
+    tree = ET.fromstring(resp.read())
+    commits = tree.findall(".//{http://www.w3.org/2005/Atom}entry")
+
+    for entry in commits:
+        link = entry.find("{http://www.w3.org/2005/Atom}link")
+        updated = entry.find("{http://www.w3.org/2005/Atom}updated")
+        assert (
+            link is not None and updated is not None and updated.text is not None
+        ), "cannot parse ATOM feed"
+        url = urlparse(link.attrib["href"])
+        commit = url.path.rsplit("/", maxsplit=1)[-1]
+        date = updated.text.split("T", maxsplit=1)[0]
+        return [Version(f"unstable-{date}", rev=commit)]
+
+    return []

nix_update/version/gitlab.py

@@ -1,8 +1,9 @@
 import json
 import re
 import urllib.request
+from datetime import datetime
 from typing import List
-from urllib.parse import ParseResult
+from urllib.parse import ParseResult, quote_plus
 
 from ..errors import VersionError
 from ..utils import info
@@ -39,3 +40,20 @@ def fetch_gitlab_versions(url: ParseResult) -> List[Version]:
     if releases == []:
         return tags
     return releases
+
+
+def fetch_gitlab_snapshots(url: ParseResult, branch: str) -> List[Version]:
+    match = GITLAB_API.match(url.geturl())
+    if not match:
+        return []
+    domain = match.group("domain")
+    project_id = match.group("project_id")
+    gitlab_url = f"https://{domain}/api/v4/projects/{project_id}/repository/commits?ref_name={quote_plus(branch)}"
+    info(f"fetch {gitlab_url}")
+    resp = urllib.request.urlopen(gitlab_url)
+    commits = json.load(resp)
+    for commit in commits:
+        date = datetime.strptime(commit["committed_date"], "%Y-%m-%dT%H:%M:%S.000%z")
+        date -= date.utcoffset()  # type: ignore[operator]
+        return [Version(date.strftime("unstable-%Y-%m-%d"), rev=commit["id"])]
+    return []

nix_update/version/version.py

@@ -7,13 +7,15 @@
 class Version:
     number: str
     prerelease: Optional[bool] = None
+    rev: Optional[str] = None
 
 
 class VersionPreference(Enum):
     STABLE = auto()
     UNSTABLE = auto()
     FIXED = auto()
     SKIP = auto()
+    BRANCH = auto()
 
     @staticmethod
     def from_str(version: str) -> "VersionPreference":
@@ -24,4 +26,6 @@ def from_str(version: str) -> "VersionPreference":
             return VersionPreference.UNSTABLE
         elif version == "skip":
             return VersionPreference.SKIP
+        elif version == "branch" or version.startswith("branch="):
+            return VersionPreference.BRANCH
         return VersionPreference.FIXED