Make wit-parser support no_std (#2415)

* make wit-parser support no_std

temporarily use forked id-arena

use alloc BTreeMap and BinaryHeap

use alloc BTreeMap and BTreeSet instead of std HashMap and HashSet

import misc items from core/alloc instead of std

use core Error, FromStr, and Debug

add std feature and gate things behind it

use different IndexMap/IndexSet for std vs no_std

misc alloc imports

add cargo check

update workspace anyhow

fixes

* use HashMap for std and IndexMap for no_std

use new instead of default

make HashMap newtype

remove collections mod

* use hashbrown instead of indexmap for internal collections usage

* remove extra file, use to_str instead of display, use hashbrown hash builder instead of ahash's and remove ashash as a dep

* use patched version of id-arena instead of fork

* PackageSourceMap is now implemented with a no_std compat PackageSources which uses String instead of PathBuf

* move fs (std) resolve functionality to resolve::fs and move resolve.rs to resolve/mod.rs

cleanup

move logic from PackageSourceMap to PackageSources

make methods public to avoid no_std flag

* enable indexmap feature std for crates that need it

* fix doc tests

Author: wilfreddenton

.github/workflows/main.yml

@@ -265,6 +265,7 @@ jobs:
       - run: cargo check --no-default-features --features wast
       - run: cargo check --no-default-features --features wit-dylib
       - run: cargo check --no-default-features -p wit-parser
+      - run: cargo check --no-default-features -p wit-parser --target x86_64-unknown-none
       - run: cargo check --no-default-features -p wit-parser --features wat
       - run: cargo check --no-default-features -p wit-parser --features serde
       - run: cargo check --no-default-features -p wit-parser --features decoding

Cargo.lock

@@ -108,7 +108,7 @@ rust-version = "1.81.0"
 
 [workspace.dependencies]
 ahash = { version = "0.8.11", default-features = false }
-anyhow = "1.0.58"
+anyhow = { version = "1.0.58", default-features = false }
 arbitrary = "1.1.0"
 bitflags = "2.5.0"
 bytesize = "2.0.0"
@@ -119,7 +119,7 @@ criterion = { version = "0.5.1", default-features = false }
 env_logger = "0.11"
 gimli = "0.31.1"
 hashbrown = { version = "0.15.2", default-features = false, features = ['default-hasher'] }
-id-arena = "2"
+id-arena = { version = "2.3.0", default-features = false }
 indexmap = { version = "2.7.0", default-features = false }
 indoc = "2.0.5"
 leb128fmt = { version = "0.1.0", default-features = false }

Cargo.toml

@@ -18,7 +18,7 @@ workspace = true
 anyhow = { workspace = true }
 heck = "0.5.0"
 im-rc = "15.1.0"
-indexmap = { workspace = true, features = ["serde"] }
+indexmap = { workspace = true, features = ["std", "serde"] }
 log = { workspace = true }
 petgraph = "0.6.2"
 serde = { workspace = true }

crates/wasm-compose/Cargo.toml

@@ -13,7 +13,7 @@ workspace = true
 [dependencies]
 anyhow = { workspace = true }
 clap = { workspace = true, optional = true }
-indexmap = { workspace = true, features = ["serde"] }
+indexmap = { workspace = true, features = ["std", "serde"] }
 serde = { workspace = true, optional = true }
 serde_derive = { workspace = true, optional = true }
 serde_json = { workspace = true, optional = true }

crates/wasm-metadata/Cargo.toml

@@ -22,7 +22,7 @@ all-features = true
 [dependencies]
 anyhow = { workspace = true }
 bitflags = "2.3.3"
-indexmap = { workspace = true }
+indexmap = { workspace = true, features = ["std"] }
 log = "0.4.17"
 serde = { workspace = true }
 serde_derive = { workspace = true }

crates/wit-component/Cargo.toml

@@ -17,7 +17,7 @@ workspace = true
 [dependencies]
 wit-parser = { workspace = true }
 wasm-encoder = { workspace = true }
-indexmap = { workspace = true }
+indexmap = { workspace = true, features = ["std"] }
 clap = { workspace = true, optional = true }
 anyhow = { workspace = true }
 
@@ -29,7 +29,7 @@ tempfile = { workspace = true }
 wit-smith = { workspace = true }
 arbitrary = { workspace = true }
 arbtest = "0.3.2"
-indexmap = { workspace = true }
+indexmap = { workspace = true, features = ["std"] }
 wit-component = { workspace = true }
 env_logger = { workspace = true }
 wasmparser = { workspace = true }

crates/wit-dylib/Cargo.toml

@@ -18,8 +18,9 @@ workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
+hashbrown = { workspace = true }
 id-arena = { workspace = true }
-indexmap = { workspace = true, features = ['std'] }
+indexmap = { workspace = true }
 log = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true, optional = true }
@@ -30,15 +31,18 @@ wasmparser = { workspace = true, optional = true, features = ['std', 'validate',
 wat = { workspace = true, optional = true, features = ['component-model'] }
 
 [features]
-default = ['serde', 'decoding']
+default = ['std', 'serde', 'decoding']
+
+# Enables use of std::path::Path and filesystem-related APIs.
+std = []
 
 # Enables support for `derive(Serialize, Deserialize)` on many structures, such
 # as `Resolve`, which can assist when encoding `Resolve` as JSON for example.
 serde = ['dep:serde', 'dep:serde_derive', 'indexmap/serde', 'serde_json']
 
 # Enables support for decoding WIT from WebAssembly. This can be done to support
 # decoding a WIT package encoded as wasm automatically.
-decoding = ['dep:wasmparser']
+decoding = ['std', 'dep:wasmparser']
 
 # Enables support for parsing the wasm text format in conjunction with the
 # `decoding` feature.

crates/wit-parser/Cargo.toml

@@ -1,4 +1,5 @@
 use crate::{Function, Handle, Int, Resolve, Type, TypeDefKind};
+use alloc::vec::Vec;
 
 /// A core WebAssembly signature with params and results.
 #[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd, Ord)]

crates/wit-parser/src/abi.rs

@@ -1,11 +1,16 @@
 use crate::{Error, PackageNotFoundError, UnresolvedPackageGroup};
+use alloc::borrow::Cow;
+use alloc::boxed::Box;
+use alloc::format;
+use alloc::string::{String, ToString};
+use alloc::vec::Vec;
 use anyhow::{Context, Result, bail};
+use core::fmt;
+use core::mem;
 use lex::{Span, Token, Tokenizer};
 use semver::Version;
-use std::borrow::Cow;
-use std::fmt;
-use std::mem;
-use std::path::{Path, PathBuf};
+#[cfg(feature = "std")]
+use std::path::Path;
 
 pub mod lex;
 
@@ -1694,7 +1699,7 @@ pub struct SourceMap {
 #[derive(Clone)]
 struct Source {
     offset: u32,
-    path: PathBuf,
+    path: String,
     contents: String,
 }
 
@@ -1711,6 +1716,7 @@ impl SourceMap {
 
     /// Reads the file `path` on the filesystem and appends its contents to this
     /// [`SourceMap`].
+    #[cfg(feature = "std")]
     pub fn push_file(&mut self, path: &Path) -> Result<()> {
         let contents = std::fs::read_to_string(path)
             .with_context(|| format!("failed to read file {path:?}"))?;
@@ -1725,7 +1731,19 @@ impl SourceMap {
     /// used to create the final parsed package namely by unioning all the
     /// interfaces and worlds defined together. Note that each file has its own
     /// personal namespace, however, for top-level `use` and such.
+    #[cfg(feature = "std")]
     pub fn push(&mut self, path: &Path, contents: impl Into<String>) {
+        self.push_str(&path.display().to_string(), contents);
+    }
+
+    /// Appends the given contents with the given source name into this source map.
+    ///
+    /// The `path` provided is not read from the filesystem and is instead only
+    /// used during error messages. Each file added to a [`SourceMap`] is
+    /// used to create the final parsed package namely by unioning all the
+    /// interfaces and worlds defined together. Note that each file has its own
+    /// personal namespace, however, for top-level `use` and such.
+    pub fn push_str(&mut self, path: &str, contents: impl Into<String>) {
         let mut contents = contents.into();
         // Guarantee that there's at least one character in these contents by
         // appending a single newline to the end. This is excluded from
@@ -1736,7 +1754,7 @@ impl SourceMap {
         let new_offset = self.offset + u32::try_from(contents.len()).unwrap();
         self.sources.push(Source {
             offset: self.offset,
-            path: path.to_path_buf(),
+            path: path.to_string(),
             contents,
         });
         self.offset = new_offset;
@@ -1762,7 +1780,7 @@ impl SourceMap {
                     src.offset,
                     self.require_f32_f64,
                 )
-                .with_context(|| format!("failed to tokenize path: {}", src.path.display()))?;
+                .with_context(|| format!("failed to tokenize path: {}", src.path))?;
                 let mut file = PackageFile::parse(&mut tokens)?;
 
                 // Filter out any nested packages and resolve them separately.
@@ -1778,10 +1796,7 @@ impl SourceMap {
                         AstItem::Package(nested_pkg) => {
                             let mut resolve = Resolver::default();
                             resolve.push(nested_pkg).with_context(|| {
-                                format!(
-                                    "failed to handle nested package in: {}",
-                                    src.path.display()
-                                )
+                                format!("failed to handle nested package in: {}", src.path)
                             })?;
 
                             nested.push(resolve.resolve()?);
@@ -1792,9 +1807,9 @@ impl SourceMap {
 
                 // With nested packages handled push this file into the
                 // resolver.
-                resolver.push(file).with_context(|| {
-                    format!("failed to start resolving path: {}", src.path.display())
-                })?;
+                resolver
+                    .push(file)
+                    .with_context(|| format!("failed to start resolving path: {}", src.path))?;
             }
             Ok(resolver.resolve()?)
         })?;
@@ -1877,7 +1892,7 @@ impl SourceMap {
  {line:4} | {snippet}
       | {marker:>0$}",
             col + 1,
-            file = src.path.display(),
+            file = src.path,
             line = line + 1,
             col = col + 1,
             marker = "^",
@@ -1898,7 +1913,7 @@ impl SourceMap {
         let (line, col) = src.linecol(start);
         format!(
             "{file}:{line}:{col}",
-            file = src.path.display(),
+            file = src.path,
             line = line + 1,
             col = col + 1,
         )
@@ -1913,8 +1928,14 @@ impl SourceMap {
     }
 
     /// Returns an iterator over all filenames added to this source map.
+    #[cfg(feature = "std")]
     pub fn source_files(&self) -> impl Iterator<Item = &Path> {
-        self.sources.iter().map(|src| src.path.as_path())
+        self.sources.iter().map(|src| Path::new(&src.path))
+    }
+
+    /// Returns an iterator over all source names added to this source map.
+    pub fn source_names(&self) -> impl Iterator<Item = &str> {
+        self.sources.iter().map(|src| src.path.as_str())
     }
 }