diff --git a/.github/workflows/publish_dragonflybsd_amd64.yml b/.github/workflows/publish_dragonflybsd_amd64.yml new file mode 100644 index 0000000..b38a728 --- /dev/null +++ b/.github/workflows/publish_dragonflybsd_amd64.yml @@ -0,0 +1,38 @@ +name: Publish for DragonflyBSD amd64 + +on: + push: + tags: + - '*' + +jobs: + test: + runs-on: macos-12 + name: Build for DragonflyBSD + steps: + - uses: actions/checkout@v3 + - name: Build in DragonflyBSD VM + id: test + uses: vmactions/dragonflybsd-vm@v0 + with: + usesh: false + mem: 8192 + prepare: | + pkg update + pkg upgrade -y + pkg install -y curl rust pkgconf openssl + rehash + + run: | + rehash + ls -lah + rustc --version + cargo build --release + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/hctl + asset_name: hctl-dragonflybsd-amd64 + tag: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/publish_freebsd.yml b/.github/workflows/publish_freebsd_amd64.yml similarity index 87% rename from .github/workflows/publish_freebsd.yml rename to .github/workflows/publish_freebsd_amd64.yml index 014c40c..21b25db 100644 --- a/.github/workflows/publish_freebsd.yml +++ b/.github/workflows/publish_freebsd_amd64.yml @@ -1,4 +1,4 @@ -name: Publish for FreeBSD +name: Publish for FreeBSD amd64 on: push: @@ -8,7 +8,7 @@ on: jobs: build_freebsd: runs-on: macos-12 - name: Build for FreeBSD-RELEASE + name: Build for FreeBSD-RELEASE repo latest steps: - uses: actions/checkout@v3 - name: Compile in FreeBSD VM @@ -16,11 +16,12 @@ jobs: uses: vmactions/freebsd-vm@v0 with: usesh: false - mem: 2048 + mem: 8192 prepare: | mkdir -p /usr/local/etc/pkg/repos cp /etc/pkg/FreeBSD.conf /usr/local/etc/pkg/repos sed -i 's/FreeBSD:/FreeBSDlatest:/g' + sed -i 's/quarterly/latest/g' echo "FreeBSD: { enabled: no }" >> /usr/local/etc/pkg/repos/FreeBSD.conf pkg update pkg upgrade -y diff --git a/.github/workflows/publish_linux_musl.yml b/.github/workflows/publish_linux_musl_amd64.yml similarity index 88% rename from .github/workflows/publish_linux_musl.yml rename to .github/workflows/publish_linux_musl_amd64.yml index 8307d57..a51e011 100644 --- a/.github/workflows/publish_linux_musl.yml +++ b/.github/workflows/publish_linux_musl_amd64.yml @@ -1,4 +1,4 @@ -name: Publish for Linux linked against musl +name: Publish for Linux musl amd64 on: push: @@ -16,7 +16,7 @@ jobs: branch: latest-stable - name: Install dependencies run: | - apk add curl rust cargo pkgconf openssl-dev + apk add curl rust cargo shell: alpine.sh --root {0} - name: Compile rootless run: | diff --git a/.github/workflows/publish_netbsd_amd64.yml b/.github/workflows/publish_netbsd_amd64.yml new file mode 100644 index 0000000..171c5ad --- /dev/null +++ b/.github/workflows/publish_netbsd_amd64.yml @@ -0,0 +1,33 @@ +name: Publish for NetBSD amd64 + +on: + push: + tags: + - '*' + +jobs: + test: + runs-on: macos-12 + name: Build for NetBSD + steps: + - uses: actions/checkout@v3 + - name: Build in NetBSD VM + id: test + uses: vmactions/netbsd-vm@v0 + with: + usesh: false + mem: 8192 + prepare: | + /usr/sbin/pkg_add curl rust pkgconf openssl + run: | + ls -lah + rustc --version + cargo build --release + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/hctl + asset_name: hctl-netbsd-amd64 + tag: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/publish_openbsd_amd64.yml b/.github/workflows/publish_openbsd_amd64.yml new file mode 100644 index 0000000..f333a33 --- /dev/null +++ b/.github/workflows/publish_openbsd_amd64.yml @@ -0,0 +1,34 @@ +name: Publish for OpenBSD amd64 + +on: + push: + tags: + - '*' + +jobs: + test: + runs-on: macos-12 + name: Build for OpenBSD + steps: + - uses: actions/checkout@v3 + - name: Compile in OpenBSD VM + id: test + uses: vmactions/openbsd-vm@v0 + with: + usesh: false + mem: 8192 + prepare: | + pkg_add curl rust pkgconf openssl + + run: | + ls -lah + rustc --version + cargo build --release + + - name: Upload binaries to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: target/release/hctl + asset_name: hctl-openbsd-amd64 + tag: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/publish_supported.yml b/.github/workflows/publish_supported.yml index a5bf78f..459a082 100644 --- a/.github/workflows/publish_supported.yml +++ b/.github/workflows/publish_supported.yml @@ -17,7 +17,7 @@ jobs: asset_name: hctl-linux-glib-amd64 - os: windows-latest artifact_name: hctl.exe - asset_name: hctl-windows-amd64 + asset_name: hctl-windows-amd64.exe - os: macos-latest artifact_name: hctl asset_name: hctl-macos-amd64 diff --git a/Cargo.toml b/Cargo.toml index d216e2e..29bda1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hctl" -version = "0.1.3" +version = "0.1.4" edition = "2021" authors = [ "Michał Szmidt" ] repository = "https://github.com/michalszmidt/hctl" diff --git a/README.md b/README.md index fa1d03d..38bb14d 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,68 @@ -# Hostlists tools cli (hctl) +``` + __ __ __ + / /_ _____ / /_ / / + / __ \ / ___/ / __/ / / + / / / // /__ / /_ / / + /_/ /_/ \___/ \__/ /_/ +``` +
+ + GitHub Last Commit + GitHub Issues + GitHub Pull Requests +
+ +## CI/CD: +### Builds +- GitHub Workflow Status (with event) FreeBSD [amd64] +- GitHub Workflow Status (with event) DragonflyBSD [amd64] +- GitHub Workflow Status (with event) NetBSD [amd64] +- GitHub Workflow Status (with event) OpenBSD [amd64] +- GitHub Workflow Status (with event) Linux musl [amd64] +- GitHub Workflow Status (with event) Linux glibc [amd64] +- GitHub Workflow Status (with event) MacOS X [amd64] +- GitHub Workflow Status (with event) Windows [amd64] + + + ## About -This is simple cli tool written in rust to manage (transform, clear from comments etc.) your hostlists +Hostlists tools cli `hctl` is cli tool written in rust to manage (merge, transform, clear from comments etc.) your hostlists from diffrent formats to selected by you! ## Features - [x] Merge hostlist - [x] Remove duplicates -- [x] Remove commented lines +- [x] Remove commented lines (default behaviour) - [x] Fetch lists directly from network (https) +- [x] YAML config - [x] Parallel processing - [x] Progressbar - [x] Rejected lines may be saved with information about source list - [x] Support for Windows encoded text files (line ending with \n\r) -- [x] Supported input formats (linewise, 127.0.0.1, 0.0.0.0) -- [x] Supported output formats (linewise, 127.0.0.1, 0.0.0.0, dnsmasq) -- [ ] Whitelists +- [x] Supported input formats (linewise, 127.0.0.1, 0.0.0.0, ~dnsmasq~, ~snort~, ~squid~, ~unbound~, ~bind~, ~netgar~, ~adblock~) +- [x] Supported output formats (linewise, 127.0.0.1, 0.0.0.0, dnsmasq, ~snort~, ~squid~, ~unbound~, ~bind~, ~netgar~, ~adblock~) +- [x] Output to stdout to be used with unix pipe stream control +- [ ] Record existance validation from DNS servers (unencrypted and over tls) +- [ ] Whitelists (enumerated, from external lists file/https) +- [ ] Deep analysis (eliminating subdomains) - [ ] Fault-tolerance, better error handling - [ ] Deep rejected analysis Jump [here](https://github.com/michalszmidt/hctl/issues/1) to feature-request issue. -## Magic behind +## Magic behind the scene There is heavy usage of rust iterators to enable lazy-reading @@ -38,4 +79,4 @@ See rules.rs if you want to embed similar solution in your app. [BSD-3-clause-no-military](https://github.com/michalszmidt/hctl/blob/main/LICENSE) TD;LR -Modified [BSD-3-clause](https://choosealicens \ No newline at end of file +Modified [BSD-3-clause](https://choosealicense.com/licenses/bsd-3-clause-clear/) that does not allow usage for military purpose. \ No newline at end of file diff --git a/src/commands.rs b/src/commands.rs index 1c5e2e4..8c4e33b 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -18,7 +18,7 @@ pub fn get_args_domain() -> Vec { .action(ArgAction::Set); let arg_out: Arg = Arg::new("out") - .help("Path to the out file [path without quotes]") + .help("Path to the out file [stdout/path without quotes]") .short('o') .long("out") .action(ArgAction::Set); diff --git a/src/main.rs b/src/main.rs index 70bf93c..19eaea8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -82,11 +82,11 @@ fn main() { match optimize.as_str() { "speed" => { (entries_len, rejected_len) = - process_parallel_list_to_file(path, out, rejected_b, format) + process_parallel_list_to_file(path, out.clone(), rejected_b, format) } "memory" => { (entries_len, rejected_len) = - process_single_list_seq_file(path, out, rejected_b, format) + process_single_list_seq_file(path, out.clone(), rejected_b, format) } _ => return, }; @@ -97,18 +97,20 @@ fn main() { return; } (entries_len, rejected_len) = - process_multiple_lists_to_file(path, out, rejected_b, format); + process_multiple_lists_to_file(path, out.clone(), rejected_b, format); } "config" => { (entries_len, rejected_len) = - config_process_lists(config, out, intro_b, rejected_b, format) + config_process_lists(config, out.clone(), intro_b, rejected_b, format) } _ => return, } - println!( - "Unique records: {}\nRemoved records: {}\n", - entries_len, rejected_len - ); + if out.as_str() != "stdout" { + println!( + "Unique records: {}\nRemoved records: {}\n", + entries_len, rejected_len + ); + } } else { unreachable!() } diff --git a/src/processing.rs b/src/processing.rs index f088818..1e22d70 100644 --- a/src/processing.rs +++ b/src/processing.rs @@ -29,11 +29,16 @@ pub fn process_parallel_list_to_file( let file_opened = file_to_lines(list_path).unwrap(); let reader = BufReader::new(file_opened); - let file_out = file_write(out_path).unwrap(); - let mut writer_out = LineWriter::new(file_out); + // let file_out = file_write(out_path).unwrap(); + // let mut writer_out = BufWriter::new(file_out); + + let mut writer_out = match out_path.as_str() { + "stdout" => Box::new(io::stdout()) as Box, + _ => Box::new(file_write(out_path).unwrap()) as Box, + }; let file_rejected = file_write("./rejected.txt".to_string()).unwrap(); - let mut writer_rejected = LineWriter::new(file_rejected); + let mut writer_rejected = BufWriter::new(file_rejected); let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new())); let mut count_entries: usize = 0; @@ -109,11 +114,14 @@ pub fn process_single_list_seq_file( let file_opened = file_to_lines(list_path).unwrap(); let reader = BufReader::new(file_opened); - let file_out = file_write(out_path).unwrap(); - let mut writer_out = LineWriter::new(file_out); + + let mut writer_out = match out_path.as_str() { + "stdout" => Box::new(io::stdout()) as Box, + _ => Box::new(file_write(out_path).unwrap()) as Box, + }; let file_rejected = file_write("./rejected.txt".to_string()).unwrap(); - let mut writer_rejected = LineWriter::new(file_rejected); + let mut writer_rejected = BufWriter::new(file_rejected); let mut set_rejected: BTreeSet = BTreeSet::new(); let mut count_entries: usize = 0; @@ -216,11 +224,13 @@ pub fn process_multiple_lists_to_file( save_rejected: bool, format: String, ) -> (usize, usize) { - let file_out = file_write(out_path).unwrap(); - let mut writer_out = LineWriter::new(file_out); + let mut writer_out = match out_path.as_str() { + "stdout" => Box::new(io::stdout()) as Box, + _ => Box::new(file_write(out_path).unwrap()) as Box, + }; let file_rejected = file_write("./rejected.txt".to_string()).unwrap(); - let mut writer_rejected = LineWriter::new(file_rejected); + let mut writer_rejected = BufWriter::new(file_rejected); let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new())); let mut count_entries: usize = 0; @@ -301,11 +311,13 @@ pub fn config_process_lists( .as_vec() .unwrap(); - let file_out = file_write(out_path).unwrap(); - let mut writer_out = LineWriter::new(file_out); + let mut writer_out = match out_path.as_str() { + "stdout" => Box::new(io::stdout()) as Box, + _ => Box::new(file_write(out_path).unwrap()) as Box, + }; let file_rejected = file_write("./rejected.txt".to_string()).unwrap(); - let mut writer_rejected = LineWriter::new(file_rejected); + let mut writer_rejected = BufWriter::new(file_rejected); let arc_mux_set_rejected = Arc::new(Mutex::new(BTreeSet::new())); let mut count_entries: usize = 0;