Allow referencing topologies from http(s) locations (#1704)

* func renaming and structuring

* added topology download from http locations

* respect dir based topology reference

* introduce schemaless toggle to allow for short urls for topology paths only

* added docs

* deepsource

clab/clab.go

@@ -131,14 +131,20 @@ func WithTopoPath(path, varsFile string) ClabOption {
 		var file string
 		var err error
 
-		switch path {
-		case "-", "stdin":
+		switch {
+		case path == "-" || path == "stdin":
 			file, err = c.readFromStdin()
 			if err != nil {
 				return err
 			}
+		// if the path is not a local file and a URL, download the file and store it in the tmp dir
+		case !utils.FileOrDirExists(path) && utils.IsHttpURL(path, true):
+			file, err = c.downloadTopoFile(path)
+			if err != nil {
+				return err
+			}
 
-		case "":
+		case path == "":
 			return fmt.Errorf("provide a path to the clab topology file")
 
 		default:
@@ -216,6 +222,19 @@ func (c *CLab) readFromStdin() (string, error) {
 	return tmpFile.Name(), nil
 }
 
+func (c *CLab) downloadTopoFile(url string) (string, error) {
+	c.TopoPaths.CreateTmpDir()
+
+	tmpFile, err := os.CreateTemp(c.TopoPaths.ClabTmpDir(), "topo-*.clab.yml")
+	if err != nil {
+		return "", err
+	}
+
+	err = utils.DownloadFile(url, tmpFile.Name())
+
+	return tmpFile.Name(), err
+}
+
 // WithNodeFilter option sets a filter for nodes to be deployed.
 // A filter is a list of node names to be deployed,
 // names are provided exactly as they are listed in the topology file.

clab/config.go

@@ -264,7 +264,7 @@ func (c *CLab) processStartupConfig(nodeCfg *types.NodeConfig) error {
 	// it contains at least one newline
 	isEmbeddedConfig := strings.Count(p, "\n") >= 1
 	// downloadable config starts with http(s)://
-	isDownloadableConfig := utils.IsHttpUri(p)
+	isDownloadableConfig := utils.IsHttpURL(p, false)
 
 	if isEmbeddedConfig || isDownloadableConfig {
 		// both embedded and downloadable configs are require clab tmp dir to be created
@@ -522,7 +522,7 @@ func (c *CLab) resolveBindPaths(binds []string, nodedir string) error {
 	return nil
 }
 
-// setClabIntfsEnvVar sets CLAB_INTFS env var for each node
+// SetClabIntfsEnvVar sets CLAB_INTFS env var for each node
 // which holds the number of interfaces a node expects to have (without mgmt interfaces).
 func (c *CLab) SetClabIntfsEnvVar() {
 	for _, n := range c.Nodes {

cmd/root.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -111,41 +112,21 @@ func getTopoFilePath(cmd *cobra.Command) error {
 	}
 
 	var err error
-	if !utils.FileExists(topo) && (utils.IsHttpUri(topo) || utils.IsGitHubShortURL(topo)) {
-		// for short github urls, prepend https://github.com
-		// note that short notation only works for github links
-		if utils.IsGitHubShortURL(topo) {
-			topo = "https://github.com/" + topo
+	// perform topology clone/fetch if the topo file is not available locally
+	if !utils.FileOrDirExists(topo) {
+		switch {
+		case git.IsGitHubOrGitLabURL(topo) || git.IsGitHubShortURL(topo):
+			topo, err = processGitTopoFile(topo)
+			if err != nil {
+				return err
+			}
+		case utils.IsHttpURL(topo, true):
+			// canonize the passed topo as URL by adding https schema if it was missing
+			if !strings.HasPrefix(topo, "http://") && !strings.HasPrefix(topo, "https://") {
+				topo = "https://" + topo
+			}
 		}
 
-		repo, err := git.NewRepo(topo)
-		if err != nil {
-			return err
-		}
-
-		// Instantiate the git implementation to use.
-		gitImpl := git.NewGoGit(repo)
-
-		// clone the repo via the Git Implementation
-		err = gitImpl.Clone()
-		if err != nil {
-			return err
-		}
-
-		// prepare the path with the repo based path
-		path := filepath.Join(repo.GetPath()...)
-		// prepend that path with the repo base directory
-		path = filepath.Join(repo.GetName(), path)
-
-		// change dir to the
-		err = os.Chdir(path)
-		if err != nil {
-			return err
-		}
-
-		// once the repo is cloned the topo file is emptied
-		// to ensure that auto find functionality can kick in
-		topo = repo.GetFilename()
 	}
 
 	// if topo or name flags have been provided, don't try to derive the topo file
@@ -167,3 +148,38 @@ func getTopoFilePath(cmd *cobra.Command) error {
 
 	return err
 }
+
+func processGitTopoFile(topo string) (string, error) {
+	// for short github urls, prepend https://github.com
+	// note that short notation only works for github links
+	if git.IsGitHubShortURL(topo) {
+		topo = "https://github.com/" + topo
+	}
+
+	repo, err := git.NewRepo(topo)
+	if err != nil {
+		return "", err
+	}
+
+	// Instantiate the git implementation to use.
+	gitImpl := git.NewGoGit(repo)
+
+	// clone the repo via the Git Implementation
+	err = gitImpl.Clone()
+	if err != nil {
+		return "", err
+	}
+
+	// prepare the path with the repo based path
+	path := filepath.Join(repo.GetPath()...)
+	// prepend that path with the repo base directory
+	path = filepath.Join(repo.GetName(), path)
+
+	// change dir to the
+	err = os.Chdir(path)
+	if err != nil {
+		return "", err
+	}
+
+	return repo.GetFilename(), err
+}

docs/cmd/deploy.md

@@ -26,6 +26,8 @@ It is possible to read the topology file from stdin by passing `-` as a value to
 
 ##### Remote topology files
 
+###### Git
+
 To simplify the deployment of labs that are stored in remote version control systems, containerlab supports the use of remote topology files for github.com and GitLab.com hosted projects.
 
 By specifying a URL to a repository or a `.clab.yml` file in a repository, containerlab will automatically clone[^1] the repository in your current directory and deploy it. If the URL points to a `.clab.yml` file, containerlab will clone the repository and deploy the lab defined in the file.
@@ -50,6 +52,19 @@ Subsequent lab operations (such as destroy) must use the filesystem path to the
         <source src="https://gitlab.com/rdodin/pics/-/wikis/uploads/5f0a7579f85c7d6af1fe05c254f42bb5/remote-labs2.mp4" type="video/mp4">
     </video>
 
+###### HTTP(S)
+
+Labs can be deployed from remote HTTP(S) URLs as well. These labs should be self-contained and not reference any external resources, like startup-config files, licenses, binds, etc.
+
+The following URL formats are supported:
+
+| Type                           | Example                                                             | Description                                           |
+| ------------------------------ | ------------------------------------------------------------------- | ----------------------------------------------------- |
+| Link to raw github gist        | https://gist.githubusercontent.com/hellt/abc/raw/def/linux.clab.yml | A file is downloaded to a temp directory and launched |
+| Link to a short schemaless URL | srlinux.dev/clab-srl                                                | A file is downloaded to a temp directory and launched |
+
+Containerlab distinct HTTP URLs from GitHub/GitLab by checking if github.com or gitlab.com is present in the URL. If not, it will treat the URL as a plain HTTP(S) URL.
+
 #### name
 
 With the global `--name | -n` flag a user sets a lab name. This value will override the lab name value passed in the topology definition file.
@@ -118,7 +133,7 @@ Read more about [node filtering](../manual/node-filtering.md) in the documentati
 
 ### Environment variables
 
-#### CLAB_RUNTIME
+#### `CLAB_RUNTIME`
 
 Default value of "runtime" key for nodes, same as global `--runtime | -r` flag described above.
 Affects all containerlab commands in the same way, not just `deploy`.
@@ -127,15 +142,15 @@ Intended to be set in environments where non-default container runtime should be
 
 Example command-line usage: `CLAB_RUNTIME=podman containerlab deploy`
 
-#### CLAB_VERSION_CHECK
+#### `CLAB_VERSION_CHECK`
 
 Can be set to "disable" value to prevent deploy command making a network request to check new version to report if one is available.
 
 Useful when running in an automated environments with restricted network access.
 
 Example command-line usage: `CLAB_VERSION_CHECK=disable containerlab deploy`
 
-#### CLAB_LABDIR_BASE
+#### `CLAB_LABDIR_BASE`
 
 To change the [lab directory](../manual/conf-artifacts.md#identifying-a-lab-directory) location, set `CLAB_LABDIR_BASE` environment variable accordingly. It denotes the base directory in which the lab directory will be created.
 

docs/htmltest-w-github.yml

@@ -16,6 +16,7 @@ IgnoreURLs:
   - https://linkedin.com/in
   - https://www.linkedin.com/in
   - mysocket.io # remove mysocket.io links until we rework to border0.com
+  - https://gist.githubusercontent.com/hellt/abc/raw/def/linux.clab.yml # test link used in docs
 IgnoreDirectoryMissingTrailingSlash: true
 IgnoreAltMissing: true
 IgnoreSSLVerify: true

docs/htmltest.yml

@@ -17,6 +17,7 @@ IgnoreURLs:
   - mysocket.io # remove mysocket.io links until we rework to border0.com
   - https://supportcenter.checkpoint.com/ # started to fail, meh.
   - https://marketplace.visualstudio.com/*
+  - https://gist.githubusercontent.com/hellt/abc/raw/def/linux.clab.yml # test link used in docs
 IgnoreDirectoryMissingTrailingSlash: true
 IgnoreAltMissing: true
 IgnoreSSLVerify: true

docs/quickstart.md

@@ -99,6 +99,9 @@ containerlab deploy # (1)!
 1. `deploy` command will automatically lookup a file matching the `*.clab.y*ml` patter to select it.  
   If you have several files and want to pick a specific one, use `--topo <path>` flag.
 
+!!!tip "Remote topology files"
+    Containerlab allows to deploy labs from files located in remote Git repositories and/or HTTP URLs. Check out deploy command [documentation](cmd/deploy.md#remote-topology-files) for more details.
+
 After a couple of seconds you will see the summary of the deployed nodes:
 
 ```

git/github.go

@@ -94,3 +94,20 @@ func IsGitHubURL(url *neturl.URL) bool {
 type GitHubRepo struct {
 	GitRepoStruct
 }
+
+// IsGitHubShortURL returns true for github-friendly short urls
+// such as srl-labs/containerlab.
+func IsGitHubShortURL(s string) bool {
+	split := strings.Split(s, "/")
+	// only 2 elements are allowed
+	if len(split) != 2 {
+		return false
+	}
+
+	// dot is not allowed in the project owner
+	if strings.Contains(split[0], ".") {
+		return false
+	}
+
+	return true
+}

git/github_test.go

@@ -230,3 +230,40 @@ func TestIsGitHubURL(t *testing.T) {
 		})
 	}
 }
+
+func TestIsGitHubShortURL(t *testing.T) {
+	tests := []struct {
+		name string
+		url  string
+		want bool
+	}{
+		{
+			name: "Valid Short URL",
+			url:  "user/repo",
+			want: true,
+		},
+		{
+			name: "Invalid Short URL - More than one slash",
+			url:  "user/repo/extra",
+			want: false,
+		},
+		{
+			name: "Invalid Short URL - Starts with http",
+			url:  "http://user/repo",
+			want: false,
+		},
+		{
+			name: "normal url in short form",
+			url:  "srlinux.dev/clab-srl",
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsGitHubShortURL(tt.url); got != tt.want {
+				t.Errorf("IsGitHubShortURL() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

git/repo.go

@@ -80,3 +80,13 @@ func NewRepo(urlPath string) (GitRepo, error) {
 
 	return r, err
 }
+
+// IsGitHubOrGitLabURL checks if the url is a github or gitlab url.
+func IsGitHubOrGitLabURL(u string) bool {
+	_url, err := url.ParseRequestURI(u)
+	if err != nil {
+		return false
+	}
+
+	return IsGitHubURL(_url) || IsGitLabURL(_url)
+}

nodes/srl/srl.go

@@ -252,7 +252,7 @@ func (s *srl) PreDeploy(_ context.Context, params *nodes.PreDeployParams) error
 		for _, fullpath := range agents {
 			basename := filepath.Base(fullpath)
 			// if it is a url extract filename from url or content-disposition header
-			if utils.IsHttpUri(fullpath) {
+			if utils.IsHttpURL(fullpath, false) {
 				basename = utils.FilenameForURL(fullpath)
 			}
 			// enforce yml extension

tests/01-smoke/12-cloned-lab.robot

@@ -15,6 +15,7 @@ ${lab2-url}             https://github.com/hellt/clab-test-repo/tree/branch1
 ${lab1-gitlab-url}      https://github.com/hellt/clab-test-repo
 ${lab1-gitlab-url2}     https://github.com/hellt/clab-test-repo/blob/main/lab1.clab.yml
 ${lab2-gitlab-url}      https://github.com/hellt/clab-test-repo/tree/branch1
+${http-lab-url}         https://gist.githubusercontent.com/hellt/66a5d8fca7bf526b46adae9008a5e04b/raw/034a542c3fbb17333afd20e6e7d21869fee6aeb5/linux.clab.yml
 ${runtime}              docker
 
 
@@ -124,6 +125,21 @@ Test lab1 with short github url
 
     Cleanup
 
+Test lab1 downloaded from https url
+    ${output} =    Process.Run Process
+    ...    sudo -E ${CLAB_BIN} --runtime ${runtime} deploy -t ${http-lab-url}
+    ...    shell=True
+
+    Log    ${output.stdout}
+    Log    ${output.stderr}
+
+    Should Be Equal As Integers    ${output.rc}    0
+
+    # check that node3 was filtered and not present in the lab output
+    Should Contain    ${output.stdout}    clab-alpine-l1
+
+    Cleanup
+
 
 *** Keywords ***
 Cleanup