diff --git a/Dockerfile.test b/Dockerfile.test index be1df89..beca948 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -7,7 +7,9 @@ ARG PLACE_COMMIT="DEV" # - Add trusted CAs for communicating with external services # - Add watchexec for running tests on change -RUN apk add --no-cache \ +RUN apk upgrade --no-cache apk \ + && \ + apk add --no-cache \ bash \ ca-certificates \ && \ diff --git a/README.md b/README.md index 4b0e000..b545ac7 100644 --- a/README.md +++ b/README.md @@ -53,3 +53,4 @@ See [`CONTRIBUTING.md`](./CONTRIBUTING.md). ## Contributors - [Caspian Baska](https://github.com/caspiano) - creator and maintainer +- [Tassja Kriek](https://github.com/tassja) - contributor and maintainer diff --git a/shard.lock b/shard.lock index 04f59b4..9273300 100644 --- a/shard.lock +++ b/shard.lock @@ -29,6 +29,10 @@ shards: git: https://github.com/maiha/crc16.cr.git version: 0.1.0 + crest: + git: https://github.com/mamantoha/crest.git + version: 1.2.1 + cron_parser: git: https://github.com/kostya/cron_parser.git version: 0.4.0 @@ -37,6 +41,10 @@ shards: git: https://github.com/spider-gazelle/crunits.git version: 1.1.0 + crystar: + git: https://github.com/naqvis/crystar.git + version: 0.2.0 + db: git: https://github.com/crystal-lang/crystal-db.git version: 0.11.0 @@ -65,18 +73,38 @@ shards: git: https://github.com/crystal-community/future.cr.git version: 1.0.0 + gitlab: + git: https://github.com/icyleaf/gitlab.cr.git + version: 0.7.0 + habitat: git: https://github.com/luckyframework/habitat.git version: 0.4.7 + halite: + git: https://github.com/icyleaf/halite.git + version: 0.12.0 + + hash_file: + git: https://github.com/tassja/hash_file.git + version: 0.4 + hound-dog: git: https://github.com/place-labs/hound-dog.git version: 2.9.0 + http-client-digest_auth: + git: https://github.com/mamantoha/http-client-digest_auth.git + version: 0.6.0 + http-params-serializable: git: https://github.com/place-labs/http-params-serializable.git version: 0.5.0 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.9.0 + ipaddress: git: https://github.com/sija/ipaddress.cr.git version: 0.2.1 @@ -105,6 +133,10 @@ shards: git: https://github.com/spider-gazelle/neuroplastic.git version: 1.11.0 + octokit: + git: https://github.com/place-labs/octokit.cr.git + version: 0.1.6 + open_api: git: https://github.com/elbywan/open_api.cr.git version: 1.3.0 diff --git a/shard.yml b/shard.yml index bb14370..4c74705 100644 --- a/shard.yml +++ b/shard.yml @@ -4,6 +4,7 @@ crystal: ">= 1.1.1" license: MIT authors: - Caspian Baska + - Tassja Kriek targets: frontends: @@ -14,6 +15,22 @@ dependencies: github: spider-gazelle/action-controller version: ~> 4.4 + crest: + github: mamantoha/crest + + crystar: + github: naqvis/crystar + + gitlab: + github: icyleaf/gitlab.cr + + hash_file: + github: tassja/hash_file + + octokit: + github: place-labs/octokit.cr + version: ~> 0.1.2 + placeos-compiler: github: placeos/compiler version: ~> 4.8 @@ -35,7 +52,6 @@ dependencies: tasker: github: spider-gazelle/tasker - development_dependencies: ameba: github: crystal-ameba/ameba diff --git a/spec/api/remotes_spec.cr b/spec/api/remotes_spec.cr new file mode 100644 index 0000000..41d16d2 --- /dev/null +++ b/spec/api/remotes_spec.cr @@ -0,0 +1,36 @@ +require "../helper" + +module PlaceOS::FrontendLoader::Api + describe Remotes do + with_server do + remotes_base = "/api/frontend-loader/v1/remotes" + it "lists releases for a given repository" do + encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader") + route = "#{remotes_base}/#{encoded_url}/releases" + result = curl("GET", route) + Array(String).from_json(result.body).includes?("v0.11.2").should be_true + end + + it "lists commits for a given repository" do + encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader") + route = "#{remotes_base}/#{encoded_url}/commits" + result = curl("GET", route) + Array(PlaceOS::FrontendLoader::Remote::Commit).from_json(result.body).should_not be_empty + end + + it "lists branches for a given repository" do + encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader") + route = "#{remotes_base}/#{encoded_url}/branches" + result = curl("GET", route) + Array(String).from_json(result.body).includes?("master").should be_true + end + + it "lists tags for a given repository" do + encoded_url = URI.encode_www_form("https://www.github.com/PlaceOS/frontend-loader") + route = "#{remotes_base}/#{encoded_url}/tags" + result = curl("GET", route) + Array(String).from_json(result.body).includes?("v1.3.0").should be_true + end + end + end +end diff --git a/spec/api/repositories_spec.cr b/spec/api/repositories_spec.cr index b3230bc..90c7911 100644 --- a/spec/api/repositories_spec.cr +++ b/spec/api/repositories_spec.cr @@ -10,6 +10,16 @@ module PlaceOS::FrontendLoader::Api commits.should_not be_empty end + it "gets the default branch if not master" do + repository = example_repository(TEST_FOLDER, uri: "https://www.github.com/placeos/backoffice") + + loader = Loader.new + loader.process_resource(:created, repository).success?.should be_true + branch = Api::Repositories.default_branch(repository.folder_name, loader: loader) + commits = Api::Repositories.commits(repository.folder_name, branch, loader: loader).not_nil! + commits.should_not be_empty + end + it "lists branches for a loaded repository" do repository = example_repository(TEST_FOLDER) loader = Loader.new @@ -30,6 +40,14 @@ module PlaceOS::FrontendLoader::Api loaded[repository.folder_name].should_not eq("HEAD") end + it "lists releases for a loaded repository" do + repository = example_repository(TEST_FOLDER) + loader = Loader.new + loader.process_resource(:created, repository).success?.should be_true + releases = Api::Repositories.releases(repository.folder_name, loader: loader).not_nil! + releases.should_not be_empty + end + describe "query" do it "does not mutate the managed repositories" do branch = "test-fixture" @@ -43,12 +61,11 @@ module PlaceOS::FrontendLoader::Api expected_path = File.join(loader.content_directory, folder) Dir.exists?(expected_path).should be_true - Compiler::Git.current_repository_commit(folder, loader.content_directory).should eq checked_out_commit + Api::Repositories.current_commit(expected_path).should eq checked_out_commit Api::Repositories.branches(folder, loader).not_nil!.should_not be_empty Api::Repositories.commits(folder, branch, loader: loader).not_nil!.should_not be_empty Api::Repositories.commits(folder, "master", loader: loader).not_nil!.should_not be_empty - - Compiler::Git.current_repository_commit(folder, loader.content_directory).should eq checked_out_commit + Api::Repositories.current_commit(expected_path).should eq checked_out_commit end end end diff --git a/spec/gitlab_spec.cr b/spec/gitlab_spec.cr new file mode 100644 index 0000000..de68bdd --- /dev/null +++ b/spec/gitlab_spec.cr @@ -0,0 +1,19 @@ +require "./helper" + +module PlaceOS::FrontendLoader + LAB_TEST_FOLDER = "test-gitlab" + describe GitLab do + before_each do + reset + end + repository = example_repository(LAB_TEST_FOLDER, uri: "https://gitlab.com/bdowney/ansible-demo/") + expected_path = File.join(TEST_DIR, repository.folder_name) + + it "downloads a GitLab archive" do + ref = PlaceOS::FrontendLoader::Remote::Reference.new(repository.uri, branch: "master") + actioner = PlaceOS::FrontendLoader::GitLab.new + actioner.download(ref: ref, path: expected_path) + Dir.exists?(File.join(expected_path, "decks")).should be_true + end + end +end diff --git a/spec/helper.cr b/spec/helper.cr index 3a2c9e0..103ce61 100644 --- a/spec/helper.cr +++ b/spec/helper.cr @@ -26,7 +26,7 @@ TEST_FOLDER = "test-repo" def example_repository( folder_name : String = UUID.random.to_s[0..8], - uri : String = "https://github.com/placeos/compiler", + uri : String = "https://www.github.com/placeos/compiler", commit : String = "HEAD", branch : String = "master" ) @@ -35,6 +35,7 @@ def example_repository( existing.uri = uri unless existing.uri == uri existing.branch = branch unless existing.branch == branch existing.commit_hash = commit unless existing.commit_hash == commit + existing.save! existing else PlaceOS::Model::Generator.repository(type: :interface).tap do |repository| @@ -43,6 +44,6 @@ def example_repository( repository.folder_name = folder_name repository.commit_hash = commit repository.branch = branch - end + end.save! end end diff --git a/spec/loader_spec.cr b/spec/loader_spec.cr index 9d7a149..ff300d1 100644 --- a/spec/loader_spec.cr +++ b/spec/loader_spec.cr @@ -85,7 +85,7 @@ module PlaceOS::FrontendLoader end it "supports changing a uri" do - expected_uri = "https://github.com/placeOS/private-drivers" + expected_uri = "https://www.github.com/placeOS/private-drivers" repository.username = "robot@place.tech" loader = Loader.new @@ -94,12 +94,12 @@ module PlaceOS::FrontendLoader repository.clear_changes_information repository.uri = expected_uri + repository.save! loader.process_resource(:updated, repository).success?.should be_true Dir.exists?(expected_path).should be_true - - url = Compiler::Git.run_git(expected_path, {"remote", "get-url", "origin"}).output.to_s - url.strip.should end_with("private-drivers") + Api::Repositories.current_repo(expected_path).should end_with("private-drivers") + File.exists?("/app/test-www/test-repo/README.md").should be_true end describe "branches" do @@ -123,14 +123,29 @@ module PlaceOS::FrontendLoader loader.process_resource(:created, repository).success?.should be_true Dir.exists?(expected_path).should be_true - Compiler::Git.current_branch(expected_path).should eq branch - + Api::Repositories.current_branch(expected_path).should eq branch repository.clear_changes_information repository.branch = updated_branch - loader.process_resource(:updated, repository).success?.should be_true Dir.exists?(expected_path).should be_true - Compiler::Git.current_branch(expected_path).should eq updated_branch + Api::Repositories.current_branch(expected_path).should eq updated_branch + end + + it "downloads a release asset" do + actioner = PlaceOS::FrontendLoader::Github.new + actioner.download_latest_asset("tassja/octokit.cr", Dir.current.to_s) + File.exists?("new_file.txt").should be_true + end + + it "downloads the release asset on repo flag" do + repository = example_repository(LAB_TEST_FOLDER, uri: "https://www.github.com/tassja/octokit.cr") + repository.release = true + repository.save! + ref = PlaceOS::FrontendLoader::Remote::Reference.from_repository(repository) + expected_path = File.join(TEST_DIR, repository.folder_name) + actioner = PlaceOS::FrontendLoader::Github.new + actioner.download(ref: ref, path: expected_path) + File.exists?(File.join(expected_path, "new_file.txt")).should be_true end end end diff --git a/src/constants.cr b/src/constants.cr index 404f0b9..7b44601 100644 --- a/src/constants.cr +++ b/src/constants.cr @@ -17,8 +17,12 @@ module PlaceOS::FrontendLoader # settings for `./placeos-frontend-loader/loader.cr` WWW = ENV["PLACE_LOADER_WWW"]? || "www" CRON = ENV["PLACE_LOADER_CRON"]? || "0 * * * *" - GIT_USER = ENV["PLACE_LOADER_GIT_USER"]? - GIT_PASS = ENV["PLACE_LOADER_GIT_PASS"]? + GIT_USER = ENV["PLACE_LOADER_GIT_USER"]? || "" + GIT_PASS = ENV["PLACE_LOADER_GIT_PASS"]? || "" + + GITLAB_TOKEN = ENV["GITLAB_TOKEN"]? || "" + + BASE_REF = "https://www.github.com/PlaceOS/www-core" # NOTE:: following used in `./placeos-frontend-loader/client.cr` # URI.parse(ENV["PLACE_LOADER_URI"]? || "http://127.0.0.1:3000") diff --git a/src/placeos-frontend-loader/api/remote/github.cr b/src/placeos-frontend-loader/api/remote/github.cr new file mode 100644 index 0000000..d23a3e7 --- /dev/null +++ b/src/placeos-frontend-loader/api/remote/github.cr @@ -0,0 +1,84 @@ +require "hash_file" +require "octokit" + +require "./remote" + +module PlaceOS::FrontendLoader + class Github < Remote + def initialize(@metadata : Metadata = Metadata.instance) + end + + private alias Remote = PlaceOS::FrontendLoader::Remote + + @github_client = Octokit.client(GIT_USER, GIT_PASS) + + # TODO: This doesn't handle missing repositories + def default_branch(repo : String) : String + url = "https://api.github.com/repos/#{repo}" + response = Crest.get(url, handle_errors: false) + + raise Exception.new("status_code for #{url} was #{response.status_code}") unless (response.success? || response.status_code == 302) + + parsed = NamedTuple(default_branch: String?).from_json(response.body) + parsed[:default_branch] || "master" + end + + # Returns the release tags for a given repo + def releases(repo : String) : Array(String) + url = "https://api.github.com/repos/#{repo}/releases" + response = Crest.get(url, handle_errors: false) + raise Exception.new("status_code for #{url} was #{response.status_code}") unless (response.success? || response.status_code == 302) + Array(NamedTuple(tag_name: String)).from_json(response.body).map(&.[:tag_name]) + end + + def download_latest_asset(repo : String, path : String) + @github_client.latest_release_asset(repo, path) + end + + def download_asset(repo : String, tag : String, path : String) + @github_client.release_asset(repo, tag, path) + end + + def url(repo_name : String) : String + "https://www.github.com/#{repo_name}" + end + + def download( + ref : Remote::Reference, + path : String, + branch : String? = "master", + hash : String? = "HEAD", + tag : String? = nil + ) + repository_uri = url(ref.repo_name) + repository_folder_name = path.split("/").last + + Git.repository_lock(repository_folder_name).write do + Log.info { { + message: "downloading repository", + repository: repository_folder_name, + branch: branch, + uri: repository_uri, + } } + + model = PlaceOS::Model::Repository.where(uri: repository_uri).first? + + if model.nil? || !model.release + hash = get_hash(hash, repository_uri, tag, branch) + temp_tar_name = Random.rand(UInt32).to_s + begin + archive_url = "https://github.com/#{ref.repo_name}/archive/#{hash}.tar.gz" + download_archive(archive_url, temp_tar_name) + extract_archive(path, temp_tar_name) + save_metadata(repository_folder_name, hash, repository_uri, branch, ref.remote_type) + rescue ex : KeyError | File::Error + Log.error(exception: ex) { "Could not download repository: #{ex.message}" } + end + else + Dir.mkdir_p(path) unless Dir.exists?(path) + self.download_latest_asset(ref.repo_name, path) + end + end + end + end +end diff --git a/src/placeos-frontend-loader/api/remote/gitlab.cr b/src/placeos-frontend-loader/api/remote/gitlab.cr new file mode 100644 index 0000000..178f4f2 --- /dev/null +++ b/src/placeos-frontend-loader/api/remote/gitlab.cr @@ -0,0 +1,83 @@ +require "hash_file" +require "gitlab" + +require "./remote" + +module PlaceOS::FrontendLoader + class GitLab < PlaceOS::FrontendLoader::Remote + def initialize(@metadata : Metadata = Metadata.instance) + end + + private alias Remote = PlaceOS::FrontendLoader::Remote + + ENDPOINT = "https://gitlab.com/api/v4" + + @gitlab_client = Gitlab.client(ENDPOINT, GITLAB_TOKEN) + + def get_repo_id(repo_name : String) + repo = URI.encode_www_form(repo_name) + @gitlab_client.project(repo)["id"].to_s.to_i + end + + def default_branch(repo : String) : String + # TODO: Determine the default from the remote + "master" + end + + # TODO: Implement + # Returns the release tags for a given repo + def releases(repo : String) : Array(String) + # repo_id = get_repo_id(repo) + # @gitlab_client.tags(repo_id).as_a.map do |value| + # value["name"].to_s + # end + [""] + end + + # Returns the tags for a given repo + def tags(repo : String) : Array(String) + repo_id = get_repo_id(repo) + @gitlab_client.tags(repo_id).as_a.map do |value| + value["name"].to_s + end + end + + def url(repo_name : String) : String + "https://gitlab.com/#{repo_name}" + end + + def download( + ref : Remote::Reference, + path : String, + branch : String? = "master", + hash : String? = "HEAD", + tag : String? = nil + ) + repository_uri = url(ref.repo_name) + repository_folder_name = path.split("/").last + + hash = get_hash(hash, repository_uri, tag, branch) + temp_tar_name = Random.rand(UInt32).to_s + + Git.repository_lock(repository_folder_name).write do + Log.info { { + message: "downloading repository", + repository: repository_folder_name, + branch: branch, + uri: repository_uri, + } } + + begin + repo_encoded = ref.repo_name.gsub("/", "%2F") + archive_url = "https://gitlab.com/api/v4/projects/#{repo_encoded}/repository/archive.tar.gz?sha=#{hash}" + + download_archive(archive_url, temp_tar_name) + extract_archive(path, temp_tar_name) + save_metadata(path, hash, ref.repo_name, branch, ref.remote_type) + rescue ex : KeyError | File::Error + Log.error(exception: ex) { "Could not download repository: #{ex.message}" } + end + end + end + end +end diff --git a/src/placeos-frontend-loader/api/remote/remote.cr b/src/placeos-frontend-loader/api/remote/remote.cr new file mode 100644 index 0000000..e3a5923 --- /dev/null +++ b/src/placeos-frontend-loader/api/remote/remote.cr @@ -0,0 +1,241 @@ +require "crest" +require "json" + +module PlaceOS::FrontendLoader + class Metadata + private getter hash_file : HashFile = HashFile + private getter lock : Mutex = Mutex.new(protection: Mutex::Protection::Reentrant) + + def initialize + hash_file.config({"base_dir" => Dir.current}) + end + + class_getter instance : Metadata do + new + end + + def get_metadata(repo_name, field) + lock.synchronize do + hash_file["#{repo_name}/metadata/#{field}"].to_s.strip + end + end + + def remote_type(repo_name) + lock.synchronize do + PlaceOS::FrontendLoader::Remote::Reference::Type.parse?(hash_file["#{repo_name}/metadata/remote_type"].to_s.strip) + end + end + + def set_metadata(repo_name, field, value) + lock.synchronize do + hash_file["#{repo_name}/metadata/#{field}"] = value.to_s + end + end + end + + abstract class PlaceOS::FrontendLoader::Remote + private alias Git = PlaceOS::Compiler::Git + private alias Remote = PlaceOS::FrontendLoader::Remote + + getter metadata : Metadata + + def initialize(@metadata : Metadata = Metadata.instance) + end + + def self.remote_for(repository_url : URI | String) : Remote + uri = repository_url.is_a?(URI) ? repository_url : URI.parse(repository_url) + {% begin %} + case uri.host.to_s + {% for remote in Reference::Type.constants %} + when .includes?(Reference::Type::{{ remote }}.to_s.downcase) + PlaceOS::FrontendLoader::{{ remote.id }}.new + {% end %} + else + raise Exception.new("Host not supported: #{repository_url}") + end + {% end %} + end + + struct Commit + include JSON::Serializable + getter commit : String + getter name : String + + def initialize(@commit, @name) + end + end + + struct Reference + include JSON::Serializable + + enum Type + GitLab + Github + end + + getter repo_name : String + getter remote_type : Reference::Type + getter branch : String + getter hash : String + getter tag : String | Nil + + def initialize(url : String | URI, @branch : String? = "master", @tag : String? = nil, @hash : String? = "HEAD") + uri = url.is_a?(URI) ? url : URI.parse(url) + @repo_name = uri.path.strip("/") + @remote_type = {% begin %} + case uri.host.to_s + {% for remote in Reference::Type.constants %} + when .includes?(Reference::Type::{{ remote }}.to_s.downcase) + Reference::Type::{{ remote.id }} + {% end %} + else + raise Exception.new("Host not supported: #{url}") + end + {% end %} + end + + def self.from_repository(repository : Model::Repository) + hash = repository.should_pull? ? "HEAD" : repository.commit_hash + self.new(url: repository.uri, branch: repository.branch, hash: hash) + end + end + + # Returns the commits for a given repo on specified branch + def commits(repo : String, branch : String) : Array(Remote::Commit) + repository_uri = url(repo) + + if branch.nil? + get_commit_hashes(repository_uri).map do |name, commit| + Remote::Commit.new( + commit: commit, + name: name.split("refs/heads/", limit: 2).last + ) + end + else + commit = Remote::Commit.new( + commit: get_commit_hashes(repository_uri, branch), + name: branch + ) + [commit] + end + end + + # Returns the branches for a given repo + def branches(repo : String) : Array(String) + get_commit_hashes(url(repo)).keys.compact_map do |name| + name.split("refs/heads/", limit: 2).last if name.includes?("refs/heads") + end.sort!.uniq! + end + + abstract def releases(repo : String) : Array(String) + + # Returns the tags for a given repo + def tags(repo : String) : Array(String) + get_commit_hashes(url(repo)).keys.compact_map do |name| + name.split("refs/tags/", limit: 2).last if name.includes?("refs/tags/") + end + end + + abstract def default_branch(repo : String) : String + + abstract def download(ref : Reference, path : String, branch : String? = "master", hash : String? = "HEAD", tag : String? = nil) + + def save_metadata(repo_path : String, hash : String, repository_uri : String, branch : String, type : Remote::Reference::Type) + metadata.set_metadata(repo_path, "current_hash", hash) + metadata.set_metadata(repo_path, "current_repo", repository_uri.split(".com/").last) + metadata.set_metadata(repo_path, "current_branch", branch) + metadata.set_metadata(repo_path, "remote_type", type) + end + + # Querying + ############################################################################################### + + # grabs the commit sha needed for repo download based on provided tag/branch or defaults to latest commit + def get_hash(hash : String, repository_uri : String, tag : String?, branch : String) : String + if hash == "HEAD" + if !tag.nil? + get_hash_by_tag(repository_uri, tag) + else + get_hash_by_branch(repository_uri, branch) + end + else + hash + end + rescue ex : KeyError + get_hash_head(repository_uri) + end + + def get_commit_hashes(repo_url : String) + uri = repo_url.gsub("www.", "") + stdout = IO::Memory.new + Process.new("git", ["ls-remote", uri], output: stdout).wait + output = stdout.to_s.split('\n') + output.compact_map do |ref| + next if ref.empty? + ref.split('\t', limit: 2).reverse + end.to_h + end + + def get_commit_hashes(repo_url : String, branch : String) + ref_hash = get_commit_hashes(repo_url) + raise KeyError.new("Branch #{branch} does not exist in repo") unless ref_hash.has_key?("refs/heads/#{branch}") + ref_hash["refs/heads/#{branch}"] + end + + private def get_hash_head(repo_url : String) + ref_hash = get_commit_hashes(repo_url) + ref_hash["HEAD"]? || ref_hash.first_key + end + + private def get_hash_by_branch(repo_url : String, branch : String) + ref_hash = get_commit_hashes(repo_url) + raise KeyError.new("Branch #{branch} does not exist in repo") unless ref_hash.has_key?("refs/heads/#{branch}") + ref_hash["refs/heads/#{branch}"] + end + + # tag = "1.9.0" + private def get_hash_by_tag(repo_url : String, tag : String) + ref_hash = get_commit_hashes(repo_url) + raise KeyError.new("Tag #{tag} does not exist in repo") unless ref_hash.has_key?("refs/tags/v#{tag}") + ref_hash["refs/tags/v#{tag}"] + end + + # Downloading + ############################################################################################### + + def download_archive(url : String, temp_tar_name : String) + Crest.get(url) do |response| + File.write(temp_tar_name, response.body_io) + end + File.new(temp_tar_name) + rescue ex : File::Error | Crest::RequestFailed + Log.error(exception: ex) { "Could not download file at #{url}" } + end + + def extract_archive(dest_path : String, temp_tar_name : String) + raise File::NotFoundError.new(message: "File #{temp_tar_name} does not exist", file: temp_tar_name) unless File.exists?(Path.new(temp_tar_name)) + if !Dir.exists?(Path.new(["./", dest_path])) + File.open(temp_tar_name) do |file| + begin + Compress::Gzip::Reader.open(file) do |gzip| + Crystar::Reader.open(gzip) do |tar| + tar.each_entry do |entry| + next if entry.file_info.directory? + parts = Path.new(entry.name).parts + parts = parts.last(parts.size > 1 ? parts.size - 1 : 0) + next if parts.size == 0 + file_path = Path.new([dest_path] + parts) + Dir.mkdir_p(file_path.dirname) unless Dir.exists?(file_path.dirname) + File.write(file_path, entry.io, perm: entry.file_info.permissions) unless Dir.exists?(file_path) + end + end + end + rescue ex : File::Error | Compress::Gzip::Error + Log.error(exception: ex) { "Could not unzip tar" } + end + end + end + File.delete(temp_tar_name) + end + end +end diff --git a/src/placeos-frontend-loader/api/remotes.cr b/src/placeos-frontend-loader/api/remotes.cr new file mode 100644 index 0000000..407359e --- /dev/null +++ b/src/placeos-frontend-loader/api/remotes.cr @@ -0,0 +1,57 @@ +module PlaceOS::FrontendLoader::Api + class Remotes < Base + base "/api/frontend-loader/v1/remotes" + Log = ::Log.for(self) + + private alias Remote = PlaceOS::FrontendLoader::Remote + + # Returns an array of releases for a repository + get "/:repository_url/releases", :releases do + url = params["repository_url"] + uri = URI.parse(URI.decode_www_form(url)) + + remote = Remote.remote_for(uri) + repo_name = uri.path.strip("/") + + releases = remote.releases(repo_name) + releases.nil? ? head :not_found : render json: releases + end + + # Returns an array of commits for a repository + get "/:repository_url/commits", :commits do + url = params["repository_url"] + uri = URI.parse(URI.decode_www_form(url)) + + remote = Remote.remote_for(uri) + repo_name = uri.path.strip("/") + branch = query_params["branch"]?.presence || "master" + + commits = remote.commits(repo_name, branch) + commits.nil? ? head :not_found : render json: commits + end + + # Returns an array of branches for a repository + get "/:repository_url/branches", :branches do + url = params["repository_url"] + uri = URI.parse(URI.decode_www_form(url)) + + remote = Remote.remote_for(uri) + repo_name = uri.path.strip("/") + + branches = remote.branches(repo_name) + branches.nil? ? head :not_found : render json: branches + end + + # Returns an array of tags for a repository + get "/:repository_url/tags", :tags do + url = params["repository_url"] + uri = URI.parse(URI.decode_www_form(url)) + + remote = Remote.remote_for(uri) + repo_name = uri.path.strip("/") + + tags = remote.tags(repo_name) + tags.nil? ? head :not_found : render json: tags + end + end +end diff --git a/src/placeos-frontend-loader/api/repositories.cr b/src/placeos-frontend-loader/api/repositories.cr index 87e5fdc..daf364a 100644 --- a/src/placeos-frontend-loader/api/repositories.cr +++ b/src/placeos-frontend-loader/api/repositories.cr @@ -1,5 +1,6 @@ require "digest/sha1" require "placeos-compiler/git" +require "hash_file" require "./base" require "../loader" @@ -9,9 +10,6 @@ module PlaceOS::FrontendLoader::Api base "/api/frontend-loader/v1/repositories" Log = ::Log.for(self) - # :nodoc: - alias Git = PlaceOS::Compiler::Git - class_property loader : Loader = Loader.instance getter loader : Loader { self.class.loader } @@ -27,18 +25,14 @@ module PlaceOS::FrontendLoader::Api commits.nil? ? head :not_found : render json: commits end - def self.commits(folder : String, branch : String, count : Int32 = 50, loader : Loader = Loader.instance) - with_query_directory(folder, loader) do |key, directory| - Git.repository_commits( - repository: key, - working_directory: directory, - count: count, - branch: branch - ) - end - rescue e - Log.error(exception: e) { "failed to fetch commmits" } - nil + def self.commits(folder : String, branch : String?, count : Int32 = 50, loader : Loader = Loader.instance) + metadata = Metadata.instance + repo = metadata.get_metadata(folder, "current_repo") + remote_type = metadata.remote_type(folder) + return unless remote_type + loader + .remote_for(remote_type) + .commits(repo, branch)[0...count] end # Returns an array of branches for a repository @@ -52,12 +46,40 @@ module PlaceOS::FrontendLoader::Api end def self.branches(folder, loader : Loader = Loader.instance) - with_query_directory(folder, loader) do |key, directory| - Git.branches(key, directory) - end - rescue e - Log.error(exception: e) { "failed to fetch branches" } - nil + metadata = Metadata.instance + repo = metadata.get_metadata(folder, "current_repo") + remote_type = metadata.remote_type(folder) + return unless remote_type + loader + .remote_for(remote_type) + .branches(repo) + end + + get "/:folder_name/releases", :releases do + folder_name = params["folder_name"] + count = (params["count"]? || 50).to_i + Log.context.set(folder: folder_name) + + releases = Repositories.releases(folder_name, count) + releases.nil? ? head :not_found : render json: releases + end + + def self.releases(folder, count : Int32 = 50, loader : Loader = Loader.instance) + metadata = Metadata.instance + repo = metadata.get_metadata(folder, "current_repo") + remote_type = metadata.remote_type(folder) + return unless remote_type + loader + .remote_for(remote_type) + .releases(repo)[0...count] + end + + def self.default_branch(folder, loader : Loader = Loader.instance) : String + metadata = Metadata.instance + repo = metadata.get_metadata(folder, "current_repo") + remote_type = metadata.remote_type(folder) + return "master" unless remote_type + loader.remote_for(remote_type).default_branch(repo) end # Returns a hash of folder name to commits @@ -73,34 +95,23 @@ module PlaceOS::FrontendLoader::Api .reject(/^\./) .select { |e| path = File.join(content_directory, e) - File.directory?(path) && File.exists?(File.join(path, ".git")) + File.directory?(path) } .each_with_object({} of String => String) { |folder_name, hash| - hash[folder_name] = Compiler::Git.current_repository_commit(folder_name, content_directory) + hash[folder_name] = Api::Repositories.current_commit(Path.new([content_directory, folder_name]).to_s) } end - # Clean repository copies to query - ########################################################################### - - class_property query_directory : String do - File.join(Dir.tempdir, "loader-queries").tap(&->Dir.mkdir_p(String)) + def self.current_branch(repository_path : String) + Metadata.instance.get_metadata(repository_path.split("/").last, "current_branch") end - def self.with_query_directory(folder, loader : Loader = Loader.instance) - authoritative_path = File.join(loader.content_directory, folder) - key = Git.repository_lock(authoritative_path).read do - remote = Git.remote(folder, loader.content_directory) - - # NOTE: url unsafe chars are removed - remote_digest = Digest::SHA1.base64digest(remote)[0..6].gsub(/(\+|\/|\=)/, "") - remote_digest.tap do |digest| - cache_path = File.join(query_directory, digest) - FileUtils.cp_r(authoritative_path, cache_path) unless Dir.exists?(cache_path) - end - end + def self.current_commit(repository_path : String) + Metadata.instance.get_metadata(repository_path.split("/").last, "current_hash") + end - yield ({key, query_directory}) + def self.current_repo(repository_path : String) + Metadata.instance.get_metadata(repository_path.split("/").last, "current_repo") end end end diff --git a/src/placeos-frontend-loader/client.cr b/src/placeos-frontend-loader/client.cr index a7ed0bf..916f06b 100644 --- a/src/placeos-frontend-loader/client.cr +++ b/src/placeos-frontend-loader/client.cr @@ -49,7 +49,7 @@ module PlaceOS::FrontendLoader params["count"] = count.to_s unless count.nil? path = "/repositories/#{folder_name}/commits?#{params}" response = get(path) - Array(NamedTuple(commit: String, date: String, author: String, subject: String)).from_json(response.body) + Array(NamedTuple(commit: String, name: String)).from_json(response.body) end # Branches for a frontend folder @@ -64,6 +64,39 @@ module PlaceOS::FrontendLoader Model::Version.from_json(get("/version").body) end + # Releases for a remote repository + def releases(repository_url : String) + encoded_url = URI.encode_www_form(repository_url) + path = "/remotes/#{encoded_url}/releases" + response = get(path) + Array(String).from_json(response.body) + end + + # Commits for a remote repository + def remote_commits(repository_url : String, branch : String) + encoded_url = URI.encode_www_form(repository_url) + params = HTTP::Params{"branch" => branch} + path = "/remotes/#{encoded_url}/commits?#{params}" + response = get(path) + Array(PlaceOS::FrontendLoader::Remote::Commit).from_json(response.body) + end + + # Branches for a remote repository + def remote_branches(repository_url : String) + encoded_url = URI.encode_www_form(repository_url) + path = "/remotes/#{encoded_url}/branches" + response = get(path) + Array(String).from_json(response.body) + end + + # Tags for a remote repository + def tags(repository_url : String) + encoded_url = URI.encode_www_form(repository_url) + path = "/remotes/#{encoded_url}/tags" + response = get(path) + Array(String).from_json(response.body) + end + ########################################################################### def initialize( diff --git a/src/placeos-frontend-loader/loader.cr b/src/placeos-frontend-loader/loader.cr index d7b6c80..188a239 100644 --- a/src/placeos-frontend-loader/loader.cr +++ b/src/placeos-frontend-loader/loader.cr @@ -5,13 +5,21 @@ require "placeos-models/repository" require "placeos-resource" require "tasker" +require "http/client" +require "crystar" + require "../constants.cr" +require "./api/remote/*" module PlaceOS::FrontendLoader class Loader < Resource(Model::Repository) Log = ::Log.for(self) private alias Git = PlaceOS::Compiler::Git + private alias Remote = PlaceOS::FrontendLoader::Remote + private alias Type = PlaceOS::FrontendLoader::Remote::Reference::Type + + private getter remotes : Hash(Type, Remote) = Hash(Type, Remote).new Habitat.create do setting content_directory : String = WWW @@ -32,10 +40,22 @@ module PlaceOS::FrontendLoader getter update_crontab : String private property update_cron : Tasker::CRON(Int64)? = nil + def remote_for(type : Type) : Remote + case type + in Type::Github + PlaceOS::FrontendLoader::Github.new + in Type::GitLab + PlaceOS::FrontendLoader::GitLab.new + end + end + def initialize( @content_directory : String = Loader.settings.content_directory, @update_crontab : String = Loader.settings.update_crontab ) + Type.values.each do |key| + @remotes[key] = remote_for(key) + end super() end @@ -52,16 +72,14 @@ module PlaceOS::FrontendLoader # Frontend loader implicitly and idempotently creates a base www protected def create_base_www - content_directory_parent = Path[content_directory].parent.to_s - Loader.clone_and_pull( - repository_folder_name: content_directory, - repository_uri: "https://github.com/PlaceOS/www-core", - content_directory: content_directory_parent, - username: Loader.settings.username, - password: Loader.settings.password, - branch: "master", - depth: 1, - ) + Model::Repository.new( + name: "PlaceOS/www-core", + repo_type: Model::Repository::Type::Interface, + folder_name: UUID.random.to_s, + uri: BASE_REF, + ).save! + base_ref = Remote::Reference.new(url: BASE_REF, branch: "master") + remotes[base_ref.remote_type].download(ref: base_ref, path: File.expand_path(content_directory)) end protected def start_update_cron : Nil @@ -78,10 +96,7 @@ module PlaceOS::FrontendLoader loaded = load_resources # Pull www (content directory) - pull_result = Git.pull(".", content_directory) - unless pull_result.success? - Log.error { "failed to pull www: #{pull_result.output}" } - end + create_base_www loaded end @@ -98,6 +113,7 @@ module PlaceOS::FrontendLoader Loader.load( repository: repository, content_directory: @content_directory, + remotes: @remotes ) in Action::Deleted # Unload the repository @@ -113,38 +129,25 @@ module PlaceOS::FrontendLoader def self.load( repository : Model::Repository, - content_directory : String + content_directory : String, + remotes : Hash(Type, Remote) ) - branch = repository.branch - username = repository.username || Loader.settings.username - password = repository.decrypt_password || Loader.settings.password - repository_commit = repository.commit_hash content_directory = File.expand_path(content_directory) repository_directory = File.expand_path(File.join(content_directory, repository.folder_name)) - if repository.uri_changed? && Dir.exists?(repository_directory) - # Reload the repository to prevent conflicting histories - unload(repository, content_directory) - end + repository_commit = repository.commit_hash - # Clone and pull the repository - clone_and_pull( - repository_folder_name: repository.folder_name, - repository_uri: repository.uri, - content_directory: content_directory, - username: username, - password: password, - branch: branch, - ) + unload(repository, content_directory) if repository.uri_changed? && Dir.exists?(repository_directory) - hash = repository.should_pull? ? "HEAD" : repository.commit_hash + # Download and extract the repository at given branch or commit + ref = Remote::Reference.from_repository(repository) - # Checkout repository to commit on the model - Git.checkout_branch(branch, repository.folder_name, content_directory) - Git._checkout(repository_directory, hash, raises: false) + current_remote = remotes[ref.remote_type] - # Grab commit for the cloned/pulled repository - checked_out_commit = Git.current_repository_commit(repository.folder_name, content_directory) + current_remote.download(ref: ref, hash: ref.hash, branch: ref.branch, path: repository_directory) + + # Grab commit for the downloaded/extracted repository + checked_out_commit = Api::Repositories.current_commit(repository_directory) # Update model commit iff... # - the repository is not held at HEAD @@ -166,12 +169,11 @@ module PlaceOS::FrontendLoader Log.info { { message: "loaded repository", commit: checked_out_commit, - branch: branch, + branch: repository.branch, repository: repository.folder_name, repository_commit: repository_commit, uri: repository.uri, } } - Resource::Result::Success end @@ -211,51 +213,5 @@ module PlaceOS::FrontendLoader end end end - - def self.clone_and_pull( - repository_folder_name : String, - repository_uri : String, - content_directory : String, - branch : String, - username : String? = nil, - password : String? = nil, - depth : Int32? = nil - ) - Git.repository_lock(repository_folder_name).write do - Log.info { { - message: "cloning repository", - repository: repository_folder_name, - branch: branch, - uri: repository_uri, - } } - - clone_result = Git.clone( - repository: repository_folder_name, - repository_uri: repository_uri, - username: username, - password: password, - working_directory: content_directory, - depth: depth, - branch: branch, - raises: true, - ) - - # Pull if already cloned and pull intended - if clone_result.output.includes?("already exists") - Log.info { { - message: "pulling repository", - repository: repository_folder_name, - branch: branch, - uri: repository_uri, - } } - - # Ensure branch is locally present - Git.fetch(repository_folder_name, content_directory) - - # Pull HEAD of branch - Git.pull(repository_folder_name, content_directory, branch: branch, raises: true) - end - end - end end end