Move forc's util and pkg related code into forc-util, forc-pkg (F…

…uelLabs#901)

* Move forc's util and pkg related code into `forc-util`, `forc-pkg`

This makes a start on FuelLabs#863 by refactoring the "package" handling code
into a `forc-pkg` crate. This new crate is solely responsible for
building, fetching, locking and updating of forc packages.

`forc-pkg` pulls in around 170 dependencies altogether, as opposed to
`forc`'s current ~460. This should be useful for downstream crates that
may want to depend on `forc` without all the bloat of the CLI tooling,
explorer, etc. The `sway-lsp` crate comes to mind.

To achieve this, some of forc's utility functionality has been moved
into `forc-util` so that it may be shared between `forc-pkg` and `forc`
itself.

* Address some clippy nits

* Update tests for parsing the default manifest and test manifest strings

* Fix dependency order in forc-pkg Cargo.toml

* Improve `forc-pkg`, `forc-util` manifest descriptions

Also tweaks the version declarations for forc-pkg dependencies.

* Specify full version for sway deps to be publish-friendly

* Specify full versions for `forc-*` deps ready for publishing

.github/workflows/ci.yml

@@ -162,6 +162,8 @@ jobs:
       run: |
         cargo install toml-cli
         ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} forc/Cargo.toml
+        ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} forc-pkg/Cargo.toml
+        ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} forc-util/Cargo.toml
         ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} sway-core/Cargo.toml
         ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} sway-fmt/Cargo.toml
         ./.github/workflows/scripts/verify_tag.sh ${{ github.ref_name }} sway-ir/Cargo.toml

Cargo.toml

@@ -4,6 +4,8 @@ members = [
     "docstrings",
     "examples/build-all-examples",
     "forc",
+    "forc-pkg",
+    "forc-util",
     "parser",
     "sway-core",
     "sway-fmt",

forc-pkg/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "forc-pkg"
+version = "0.6.0"
+authors = ["Fuel Labs <[email protected]>"]
+edition = "2021"
+homepage = "https://fuel.network/"
+license = "Apache-2.0"
+repository = "https://github.com/FuelLabs/sway"
+description = "Building, locking, fetching and updating sway projects as Forc packages."
+
+[dependencies]
+anyhow = "1"
+forc-util = { version = "0.6.0", path = "../forc-util" }
+git2 = "0.14"
+petgraph = { version = "0.6", features = ["serde-1"] }
+semver = { version = "1.0", features = ["serde"] }
+serde = { version = "1.0", features = ["derive"] }
+sway-core = { version = "0.6.0", path = "../sway-core" }
+sway-types = { version = "0.6.0", path = "../sway-types" }
+sway-utils = { version = "0.6.0", path = "../sway-utils" }
+toml = "0.5"
+url = { version = "2.2", features = ["serde"] }

forc-pkg/src/lib.rs

@@ -0,0 +1,14 @@
+//! Building, locking, fetching and updating sway projects as Forc packages.
+//!
+//! A forc package represents a Sway project with a `Forc.toml` manifest file declared at its root.
+//! The project should consist of one or more Sway modules under a `src` directory. It may also
+//! declare a set of forc package dependencies within its manifest.
+
+pub mod lock;
+pub mod manifest;
+mod pkg;
+
+pub use lock::Lock;
+pub use manifest::Manifest;
+#[doc(inline)]
+pub use pkg::*;

forc/src/lock.rs → forc-pkg/src/lock.rs

@@ -1,5 +1,6 @@
 use crate::pkg;
 use anyhow::{anyhow, Result};
+use forc_util::{println_green, println_red};
 use petgraph::{visit::EdgeRef, Direction};
 use serde::{Deserialize, Serialize};
 use std::{
@@ -11,21 +12,21 @@ use std::{
 
 /// The graph of pinned packages represented as a toml-serialization-friendly structure.
 #[derive(Debug, Default, Deserialize, Serialize)]
-pub(crate) struct Lock {
+pub struct Lock {
     // Named `package` so that each entry serializes to lock file under `[[package]]` like cargo.
     pub(crate) package: BTreeSet<PkgLock>,
 }
 
 /// Packages that have been removed and added between two `Lock` instances.
 ///
 /// The result of `new_lock.diff(&old_lock)`.
-pub(crate) struct Diff<'a> {
-    pub(crate) removed: BTreeSet<&'a PkgLock>,
-    pub(crate) added: BTreeSet<&'a PkgLock>,
+pub struct Diff<'a> {
+    pub removed: BTreeSet<&'a PkgLock>,
+    pub added: BTreeSet<&'a PkgLock>,
 }
 
 #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Deserialize, Serialize)]
-pub(crate) struct PkgLock {
+pub struct PkgLock {
     pub(crate) name: String,
     // TODO: Cargo *always* includes version, whereas we don't even parse it when reading a
     // project's `Manifest` yet. If we decide to enforce versions, we'll want to remove the
@@ -169,3 +170,30 @@ fn pkg_unique_string(name: &str, source: Option<&str>) -> String {
         Some(s) => format!("{} {}", name, s),
     }
 }
+
+pub fn print_diff(proj_name: &str, diff: &Diff) {
+    print_removed_pkgs(proj_name, diff.removed.iter().cloned());
+    print_added_pkgs(proj_name, diff.added.iter().cloned());
+}
+
+pub fn print_removed_pkgs<'a, I>(proj_name: &str, removed: I)
+where
+    I: IntoIterator<Item = &'a PkgLock>,
+{
+    for pkg in removed {
+        if pkg.name != proj_name {
+            let _ = println_red(&format!("  Removing {}", pkg.unique_string()));
+        }
+    }
+}
+
+pub fn print_added_pkgs<'a, I>(proj_name: &str, removed: I)
+where
+    I: IntoIterator<Item = &'a PkgLock>,
+{
+    for pkg in removed {
+        if pkg.name != proj_name {
+            let _ = println_green(&format!("    Adding {}", pkg.unique_string()));
+        }
+    }
+}

forc-pkg/src/manifest.rs

@@ -0,0 +1,136 @@
+use anyhow::anyhow;
+use forc_util::validate_name;
+use serde::{Deserialize, Serialize};
+use std::{
+    collections::BTreeMap,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+use sway_utils::constants;
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct Manifest {
+    pub project: Project,
+    pub network: Option<Network>,
+    pub dependencies: Option<BTreeMap<String, Dependency>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct Project {
+    #[deprecated = "use the authors field instead, the author field will be removed soon."]
+    pub author: Option<String>,
+    pub authors: Option<Vec<String>>,
+    pub name: String,
+    pub organization: Option<String>,
+    pub license: String,
+    #[serde(default = "default_entry")]
+    pub entry: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct Network {
+    #[serde(default = "default_url")]
+    pub url: String,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum Dependency {
+    /// In the simple format, only a version is specified, eg.
+    /// `package = "<version>"`
+    Simple(String),
+    /// The simple format is equivalent to a detailed dependency
+    /// specifying only a version, eg.
+    /// `package = { version = "<version>" }`
+    Detailed(DependencyDetails),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct DependencyDetails {
+    pub(crate) version: Option<String>,
+    pub(crate) path: Option<String>,
+    pub(crate) git: Option<String>,
+    pub(crate) branch: Option<String>,
+    pub(crate) tag: Option<String>,
+}
+
+impl Manifest {
+    pub const DEFAULT_ENTRY_FILE_NAME: &'static str = "main.sw";
+
+    /// Given a path to a `Forc.toml`, read it and construct a `Manifest`.
+    ///
+    /// This also `validate`s the manifest, returning an `Err` in the case that invalid names,
+    /// fields were used.
+    pub fn from_file(path: &Path) -> anyhow::Result<Self> {
+        let manifest = std::fs::read_to_string(path)
+            .map_err(|e| anyhow!("failed to read manifest at {:?}: {}", path, e))?;
+        let manifest: Self =
+            toml::from_str(&manifest).map_err(|e| anyhow!("failed to parse manifest: {}.", e))?;
+        manifest.validate()?;
+        Ok(manifest)
+    }
+
+    /// Given a directory to a forc project containing a `Forc.toml`, read the manifest.
+    ///
+    /// This is short for `Manifest::from_file`, but takes care of constructing the path to the
+    /// file.
+    pub fn from_dir(manifest_dir: &Path) -> anyhow::Result<Self> {
+        let file_path = manifest_dir.join(constants::MANIFEST_FILE_NAME);
+        Self::from_file(&file_path)
+    }
+
+    /// Validate the `Manifest`.
+    ///
+    /// This checks the project and organization names against a set of reserved/restricted
+    /// keywords and patterns.
+    pub fn validate(&self) -> anyhow::Result<()> {
+        validate_name(&self.project.name, "package name")?;
+        if let Some(ref org) = self.project.organization {
+            validate_name(org, "organization name")?;
+        }
+        Ok(())
+    }
+
+    /// Given the directory in which the file associated with this `Manifest` resides, produce the
+    /// path to the entry file as specified in the manifest.
+    pub fn entry_path(&self, manifest_dir: &Path) -> PathBuf {
+        manifest_dir
+            .join(constants::SRC_DIR)
+            .join(&self.project.entry)
+    }
+
+    /// Produces the string of the entry point file.
+    pub fn entry_string(&self, manifest_dir: &Path) -> anyhow::Result<Arc<str>> {
+        let entry_path = self.entry_path(manifest_dir);
+        let entry_string = std::fs::read_to_string(&entry_path)?;
+        Ok(Arc::from(entry_string))
+    }
+
+    /// Produce an iterator yielding all listed dependencies.
+    pub fn deps(&self) -> impl Iterator<Item = (&String, &Dependency)> {
+        self.dependencies
+            .as_ref()
+            .into_iter()
+            .flat_map(|deps| deps.iter())
+    }
+
+    /// Produce an iterator yielding all `Detailed` dependencies.
+    pub fn deps_detailed(&self) -> impl Iterator<Item = (&String, &DependencyDetails)> {
+        self.deps().filter_map(|(name, dep)| match dep {
+            Dependency::Detailed(ref det) => Some((name, det)),
+            Dependency::Simple(_) => None,
+        })
+    }
+}
+
+fn default_entry() -> String {
+    Manifest::DEFAULT_ENTRY_FILE_NAME.to_string()
+}
+
+fn default_url() -> String {
+    constants::DEFAULT_NODE_URL.into()
+}