Add lib fetch command (#1366)

* Add lib fetch command

* fix node I hope

* disable buildtime initialization

* remove static option

build.sbt

@@ -151,13 +151,8 @@ lazy val cli = (project in file("cli"))
         munit.value % Test,
         munitScalaCheck.value % Test
       ),
-    nativeImageOptions ++= {
-      val common =
-        List("--no-fallback", "--verbose", "--initialize-at-build-time")
-      if (Option(System.getProperty("os.name")).exists(_.contains("Mac OS")))
-        common
-      else ("--static" :: common)
-    },
+    // static linking doesn't work with macos or with linux http4s on the path
+    nativeImageOptions ++= List("--no-fallback", "--verbose"),
     nativeImageVersion := "22.3.0"
   )
   .dependsOn(protoJVM, coreJVM % "compile->compile;test->test")
@@ -231,7 +226,7 @@ lazy val cliJS =
       name := "bosatsu-clijs",
       assembly / test := {},
       mainClass := Some("org.bykn.bosatsu.tool.Fs2Main"),
-      libraryDependencies ++= Seq(fs2core.value, fs2io.value, catsEffect.value, http4sEmber.value)
+      libraryDependencies ++= Seq(fs2core.value, fs2io.value, catsEffect.value, http4sCore.value, http4sEmber.value)
     )
     .jsSettings(
       scalaJSLinkerConfig ~= { _.withModuleKind(ModuleKind.CommonJSModule) },

cli/src/main/scala/org/bykn/bosatsu/IOPlatformIO.scala

@@ -85,6 +85,8 @@ object IOPlatformIO extends PlatformIO[IO, JPath] {
   override def moduleIOMonad: MonadError[IO, Throwable] =
     cats.effect.IO.asyncForIO
 
+  override val parallelF: cats.Parallel[IO] = IO.parallelForIO
+
   private val parResource: Resource[IO, Par.EC] =
     Resource.make(IO(Par.newService()))(es => IO(Par.shutdownService(es)))
       .map(Par.ecFromService(_))
@@ -191,12 +193,14 @@ object IOPlatformIO extends PlatformIO[IO, JPath] {
             .flatMap(h => fs2.Stream.emit(algo.finishHash(h)))
         }
 
+    val parent = Option(path.getParent)
     val tempFileRes = filesIO.tempFile(
-      dir = Option(path.getParent: Path),
+      dir = parent,
       prefix = s"${algo.name}_${hash.hex.take(12)}",
       suffix = "temp"
     )
 
+    parent.traverse_(p => filesIO.createDirectories(Fs2Path.fromNioPath(p))) *>
     (clientResource,
       tempFileRes,
       Resource.eval(IO(Uri.unsafeFromString(uri)))

cliJS/src/main/scala/org/bykn/bosatsu/Fs2PlatformIO.scala

@@ -17,6 +17,8 @@ object Fs2PlatformIO extends PlatformIO[IO, Path] {
   def moduleIOMonad: MonadError[IO, Throwable] =
     IO.asyncForIO
 
+  override val parallelF: cats.Parallel[IO] = IO.parallelForIO
+
   val pathArg: Argument[Path] =
     new Argument[Path] {
       def read(string: String) =
@@ -168,6 +170,7 @@ object Fs2PlatformIO extends PlatformIO[IO, Path] {
       permissions = None
     )
 
+    path.parent.traverse_(FilesIO.createDirectories(_)) *>
     (clientResource,
       tempFileRes,
       Resource.eval(IO(Uri.unsafeFromString(uri)))

core/src/main/scala/org/bykn/bosatsu/MemoryMain.scala

@@ -157,6 +157,7 @@ object MemoryMain {
         type F[A] = StateT[G, State, A]
         type Path = Chain[String]
         def moduleIOMonad: MonadError[F, Throwable] = catsDefaultME
+        val parallelF: cats.Parallel[F] = cats.Parallel.identity[F]
         def pathOrdering = Chain.catsDataOrderForChain[String].toOrdering
         val pathArg: Argument[Path] =
           new Argument[Path] {

core/src/main/scala/org/bykn/bosatsu/PlatformIO.scala

@@ -2,7 +2,7 @@ package org.bykn.bosatsu
 
 import _root_.bosatsu.{TypedAst => proto}
 
-import cats.{MonadError, Show}
+import cats.{MonadError, Parallel, Show}
 import cats.data.ValidatedNel
 import cats.parse.{Parser0 => P0}
 import com.monovore.decline.Argument
@@ -15,6 +15,8 @@ import org.bykn.bosatsu.hashing.{Algo, Hashed, HashValue}
 
 trait PlatformIO[F[_], Path] {
   implicit def moduleIOMonad: MonadError[F, Throwable]
+  implicit def parallelF: Parallel[F]
+
   implicit def pathArg: Argument[Path]
   implicit def pathOrdering: Ordering[Path]
 
@@ -71,6 +73,12 @@ trait PlatformIO[F[_], Path] {
 
   def fsDataType(p: Path): F[Option[PlatformIO.FSDataType]]
 
+  def fileExists(p: Path): F[Boolean] =
+    fsDataType(p).map {
+      case Some(PlatformIO.FSDataType.File) => true
+      case _ => false
+    }
+
   def resolve(p: Path, child: String): Path
   def resolve(p: Path, child: Path): Path
   def relativize(prefix: Path, deeper: Path): Option[Path]

core/src/main/scala/org/bykn/bosatsu/hashing/Algo.scala

@@ -65,6 +65,16 @@ object Algo {
         def algo = alg
         def value = v
       }
+
+    implicit class WithAlgoHashValue(private val hashValue: WithAlgo[HashValue]) extends AnyVal {
+      def toIdent: String = hashValue.value.toIdent(hashValue.algo)
+    }
+
+    implicit val ordWithAlgoHash: cats.Order[WithAlgo[HashValue]] =
+      cats.Order.by(_.toIdent)
+
+    implicit val orderingWithAlgoHash: Ordering[WithAlgo[HashValue]] =
+      Ordering.by(_.toIdent)
   }
 
   def hashBytes[A](bytes: Array[Byte])(implicit algo: Algo[A]): HashValue[A] =

core/src/main/scala/org/bykn/bosatsu/library/Command.scala

@@ -1,9 +1,15 @@
 package org.bykn.bosatsu.library
 
-import cats.MonoidK
-import org.bykn.bosatsu.{Json, PlatformIO}
-import org.bykn.bosatsu.tool.{CliException, Output}
+import cats.{Monad, MonoidK}
 import com.monovore.decline.Opts
+import org.bykn.bosatsu.tool.{CliException, Output}
+import org.bykn.bosatsu.hashing.{Algo, HashValue}
+import org.bykn.bosatsu.{Json, PlatformIO}
+import org.typelevel.paiges.Doc
+import scala.collection.immutable.SortedMap
+
+import _root_.bosatsu.{TypedAst => proto}
+
 import cats.syntax.all._
 
 object Command {
@@ -52,6 +58,156 @@ object Command {
     def confPath(root: P, name: Name): P =
       platformIO.resolve(root, s"${name.name}_conf.json")
 
+    def fetchAllDeps(casDir: P, deps: List[proto.LibDependency]): F[Doc] = {
+      import platformIO.parallelF
+
+      // Right(true): succeeded to download
+      // Right(false): succeeded with cached
+      // Left: failed to download or add to cache
+      type DownloadRes = Either[Throwable, Boolean]
+      type FetchState = SortedMap[(String, Version), SortedMap[Algo.WithAlgo[HashValue], DownloadRes]]
+
+      def hashes(dep: proto.LibDependency): List[Algo.WithAlgo[HashValue]] =
+        for {
+          desc <- dep.desc.toList
+          hash <- desc.hashes
+          hashValue <- Algo.parseIdent.parseAll(hash).toOption.toList
+        } yield hashValue
+
+      def depUris(dep: proto.LibDependency): List[String] =
+        dep.desc.toList.flatMap(_.uris)
+
+      def casPaths(dep: proto.LibDependency): SortedMap[Algo.WithAlgo[HashValue], P] =
+        hashes(dep).map { withAlgo =>
+          val algoName = withAlgo.algo.name  
+          val hex1 = withAlgo.value.hex.take(2)
+          val hex2 = withAlgo.value.hex.drop(2)
+
+          val path = platformIO.resolve(casDir, algoName :: hex1 :: hex2 :: Nil)
+
+          (withAlgo, path)
+        }
+        .to(SortedMap)
+
+      def libFromCas(dep: proto.LibDependency): F[Option[proto.Library]] =
+        casPaths(dep).values.toList.collectFirstSomeM { path =>
+          platformIO.fileExists(path).flatMap {
+            case true => platformIO.readLibrary(path).map(h => Some(h.arg))
+            case false => Monad[F].pure(None)
+          }  
+        }
+
+      def fetchIfNeeded(dep: proto.LibDependency): F[SortedMap[Algo.WithAlgo[HashValue], DownloadRes]] = {
+        val paths = casPaths(dep)
+        val uris = depUris(dep) 
+
+        paths.transform { (hashValue, path) =>
+          platformIO
+            .fileExists(path)
+            .flatMap {
+              case true => Monad[F].pure(Right(false)).widen[DownloadRes]
+              case false => {
+                // We need to download
+                uris.foldM((List.empty[(String, Throwable)], false)) {
+                  case ((fails, false), uri) =>
+                    platformIO.fetchHash(hashValue.algo, hashValue.value, path, uri)
+                      .attempt
+                      .map {
+                        case Right(_) => ((fails, true))
+                        case Left(e) => (((uri, e) :: fails, false))
+                      }
+                  case (done, _) => Monad[F].pure(done)
+                }
+                .map {
+                  case (_, true) =>
+                    // we were able to download
+                    Right(true)
+                  case (fails, false) =>
+                    // couldn't download
+                    Left(CliException(s"download failure: ${dep.name} with ${fails.size} fails.",
+                      if (fails.isEmpty) Doc.text(s"failed to fetch ${dep.name} with no uris.")
+                      else {
+                      Doc.text(s"failed to fetch ${dep.name} with ${fails.size} fails:") +
+                      (Doc.line + Doc.intercalate(Doc.line + Doc.line, fails.map { case (uri, f) =>
+                        Doc.text(s"uri=$uri failed with ${f.getMessage}")
+                      })).nested(4)   
+                    }))
+                }
+              }
+              .widen[DownloadRes]
+            }
+          }
+          .parSequence
+      }
+
+      def versionOf(dep: proto.LibDependency): Version =
+        dep.desc.flatMap(_.version) match {
+          case None => Version.zero
+          case Some(v) => Version.fromProto(v)
+        }
+
+      def step(
+        fetched: FetchState,
+        batch: List[proto.LibDependency]): F[(FetchState, List[proto.LibDependency])] =
+        batch.parTraverse { dep =>
+          fetchIfNeeded(dep).map { fetchMap => (dep, fetchMap) }
+        }
+        .flatMap { thisFetched =>
+          val nextFetched = fetched ++ thisFetched.map { case (dep, fm) => (dep.name, versionOf(dep)) -> fm } 
+
+          val nextBatchF: F[List[proto.LibDependency]] =
+            thisFetched.parTraverse { case (dep, _) =>
+              libFromCas(dep)    
+            }  
+            .map { fetchedLibsOpt =>
+              val fetchedDeps = fetchedLibsOpt.flatMap {
+                case None => Nil
+                case Some(dep) =>
+                  // we will find the transitivies by walking them
+                  dep.publicDependencies.toList ::: dep.privateDependencies.toList
+              }
+
+              fetchedDeps.filterNot { dep =>
+                nextFetched.contains((dep.name, versionOf(dep)))  
+              }
+            }
+
+          nextBatchF.map((nextFetched, _))
+        }
+
+      moduleIOMonad.tailRecM((SortedMap.empty: FetchState, deps)) { case (fetched, deps) =>
+        step(fetched, deps).map {
+          case (state, Nil) => Right(state)
+          case next => Left(next)
+        }
+      }
+      .flatMap { fs =>
+
+        val depStr = if (fs.size == 1) "dependency" else "dependencies"
+        val header = Doc.text(s"fetched ${fs.size} transitive ${depStr}.")
+
+        val resultDoc = header + Doc.line + Doc.intercalate(Doc.hardLine, fs.toList.map { case ((n, v), hashes) =>
+          val sortedHashes = hashes.toList.sortBy(_._1.toIdent)
+          val hashDoc = Doc.intercalate(Doc.comma + Doc.line, sortedHashes.map { case (wh, msg) =>
+            val ident = wh.toIdent
+            msg match {
+              case Right(true) => Doc.text(show"fetched $ident")
+              case Right(false) => Doc.text(show"cached $ident")
+              case Left(err) => Doc.text(show"failed: $ident ${err.getMessage}")
+            }
+          })
+
+          Doc.text(show"$n $v:") + (Doc.line + hashDoc).nested(4).grouped
+        })
+
+        val success = fs.forall { case (_, dl) => dl.forall { case (_, res) => res.isRight } }
+        if (success) moduleIOMonad.pure(resultDoc)
+        else moduleIOMonad.raiseError(
+          CliException("failed to fetch", err = resultDoc)
+        )
+      }
+    }
+
     val initCommand = 
       Opts.subcommand("init", "initialize a config") {
         (Opts.option[Name]("name", "name of the library"),
@@ -151,8 +307,8 @@ object Command {
         )
           .mapN { (fpnp, packs, deps, optOut, prevLibPath, readGitSha) =>
             for {
-              pnp <- fpnp
               gitSha <- readGitSha
+              pnp <- fpnp
               (gitRoot, name, confPath) = pnp
               conf <- readLibConf(name, confPath)
               outPath <- optOut match {
@@ -168,6 +324,32 @@ object Command {
           }
       }
 
-    MonoidK[Opts].combineAllK(initCommand :: listCommand :: assembleCommand :: Nil)
+    val casDirOpts: Opts[P => P] =
+      Opts.option[P]("cas_dir", "the path to the cas/ directory, by default .bosatsuc/cas/ in git root")
+        .orNone
+        .map {
+          case None => { root => platformIO.resolve(root, ".bosatsuc" :: "cas" :: Nil) }
+          case Some(d) => { _ => d }
+        }
+
+    val fetchCommand =
+      Opts.subcommand("fetch", "download all transitive deps into the content storage.") {
+        (rootAndName, casDirOpts).mapN { (fpnp, casDirFn) =>
+          for {
+            pnp <- fpnp
+            (gitRoot, name, confPath) = pnp
+            conf <- readLibConf(name, confPath)
+            casDir = casDirFn(gitRoot)
+            msg <- fetchAllDeps(casDir, conf.publicDeps ::: conf.privateDeps)
+          } yield (Output.Basic(msg, None): Output[P])
+        }
+      }
+
+    MonoidK[Opts].combineAllK(
+      initCommand ::
+      listCommand ::
+      assembleCommand ::
+      fetchCommand ::
+      Nil)
   }
 }

project/Dependencies.scala

@@ -15,7 +15,8 @@ object Dependencies {
   lazy val jawnAst = Def.setting("org.typelevel" %%% "jawn-ast" % "1.6.0")
   lazy val jython = Def.setting("org.python" % "jython-standalone" % "2.7.4")
   lazy val http4sBlaze = Def.setting("org.http4s" %% "http4s-blaze-client" % "0.23.17")
-  lazy val http4sEmber = Def.setting("org.http4s" %% "http4s-ember-client" % "0.23.30")
+  lazy val http4sCore = Def.setting("org.http4s" %%% "http4s-core" % "0.23.30")
+  lazy val http4sEmber = Def.setting("org.http4s" %%% "http4s-ember-client" % "0.23.30")
   lazy val munit = Def.setting("org.scalameta" %%% "munit" % "1.0.4")
   lazy val munitScalaCheck =
     Def.setting("org.scalameta" %%% "munit-scalacheck" % "1.0.0")