From 4dc9e2d69964768238c819986633c25b7b8fb05e Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Sat, 28 Oct 2023 14:11:19 -0600 Subject: [PATCH] fix(crabz): fix installers, update README --- crabz/README.md | 128 ++++++++++++++++++++++++++-------------------- crabz/install.ps1 | 49 +++++++----------- crabz/install.sh | 73 +++++++++++++------------- crabz/releases.js | 19 +++++-- 4 files changed, 144 insertions(+), 125 deletions(-) diff --git a/crabz/README.md b/crabz/README.md index 7d615a7..352b017 100644 --- a/crabz/README.md +++ b/crabz/README.md @@ -1,98 +1,116 @@ --- +title: Crabz +homepage: https://github.com/sstadick/crabz +tagline: | + crabz: multi-threaded gzip (like pigz, but in Rust) +--- -### Title: Crabz -Homepage: [Crabz on GitHub](https://github.com/sstadick/crabz) -Tagline: | - Crabz: A cross-platform, fast, compression and decompression tool written in Rust. +To update or switch versions, run `webi crabz@stable` (or `@0.8`, `@beta`, etc). ---- +## Cheat Sheet -### How to Install or Switch Versions +> `crabz` brings the power of multi-core compression to gzip and deflate. \ +> (and a few other formats + other useful features) -To update or switch versions, you can use package managers like Homebrew, or -languages-specific package managers like Cargo. For example, to install using -Cargo, run: +gzip, faster. -```bash -cargo install crabz +```sh +crabz -I ./example.json +crabz -d -I ./example.json.gz ``` +```text +Compressing (gzip) with 8 threads at compression level 6. +Decompressing (gzip) with 8 threads available. +``` + +## Table of Contents + +- Files +- Tar +- Other Formats + ### Files These are the files/directories that are created and/or modified with this install: ```text -~/.cargo/bin/crabz -~/.local/bin/crabz (if installed via Homebrew/Linuxbrew) +~/.config/envman/PATH.env +~/.local/bin/crabz ``` -### Cheat Sheet +#### How to Optimize -> `Crabz` is a fast, cross-platform compression and decompression tool. It -> supports multiple compression formats like Gzip, Zlib, Mgzip, BGZF, Raw -> Deflate, and Snap. - -#### Basic Usage - -To show help: +| Flag | Value | Comments | +| ----------------------------- | ----- | ------------------------------------------ | +| `-l`, `--compression-level` | 1-9 | higher is slower | +| `-p`, `--compression-threads` | 8 | set to the number of available cores | +| | | (but no more than 4 for decompression) | +| `-P`, `--pin-at` | 0 | pin to physical cores, starting at N | +| | | (so 4 threads starting at 0 is 0, 1, 2, 3) | ```sh -crabz -h +crabz -l 9 -p 8 -I ./example.tar + +crabz -d -p 4 -I ./example.tar.gz ``` -#### Compressing a File +#### How to use with Tar -By default, `crabz` uses gzip compression: +Tar and then compress: ```sh -crabz [FILE] +tar cv ./example/ | crabz -o ./example.tar.gz ``` -#### Decompressing a File - -To decompress a file, use the `-d` flag: +Or decompress and then untar: ```sh -crabz -d [FILE] +crabz -d ./example.tar.gz | tar xv ``` -#### Specifying Compression Level +#### How to use with other formats -To specify a compression level between 1 and 9: +`crabz` supports most of the _LZ77 with Huffman coding_ compression formats: + +| Format | Extension | Notes | +| ---------------- | --------- | ------------------------------------ | +| `gzip` | `.gz` | of GNU fame | +| [`bgzf`][bgzf] | `.gz` | supports random-access decompression | +| [`mgzip`][mgzip] | `.gz` | of python fame | +| `zlib` | `.zz` | of PNG fame, also `.z` | +| [`snap`][snap] | `.sz` | of LevelDB and MongoDB fame | +| `deflate` | `.gz` | the O.G. LZ77 | ```sh -crabz -l 9 [FILE] +crabz --format mgzip -I ./example.tar ``` -#### Using Multiple Threads - -To specify the number of threads: - ```sh -crabz -p 4 [FILE] +# DO NOT decompress in-place +crabz --format mgzip -d ./example.tar.gz -o ./example.tar + +# verify before removing the original +tar tf ./example.tar ``` -### Add More Features +⚠️ **Warnings**: -`Crabz` also allows you to pin threads to specific cores for performance -optimization, perform in-place compression/decompression, and choose from -various formats. +- DO NOT deflate in-place with non-standard formats: \ + Although `gunzip` will work correctly on files compressed with `mgzip` or + `bgzf`, some combinations (ex: decompressing from `mgzip` with `bgzf`) could + result in corruption! +- `tar xvf` and `gzip -l` may report incorrect information, even though `gunzip` + will work -For pinning threads: +See also: -```sh -crabz -P 0 [FILE] -``` +- https://dev.to/biellls/compression-clearing-the-confusion-on-zip-gzip-zlib-and-deflate-15g1 -For in-place compression: +(p.s. `zip` isn't in the list because it's a container format like `tar`, not a +zip format) -```sh -crabz -I [FILE] -``` - -To choose a format: - -```sh -crabz -f zlib [FILE] -``` +[snap]: https://github.com/google/snappy/blob/main/format_description.txt +[bgzf]: https://samtools.github.io/hts-specs/SAMv1.pdf +[mgzip]: https://pypi.org/project/mgzip/ diff --git a/crabz/install.ps1 b/crabz/install.ps1 index c605600..6b5cf7c 100644 --- a/crabz/install.ps1 +++ b/crabz/install.ps1 @@ -1,13 +1,14 @@ #!/usr/bin/env pwsh -################## +################# # Install crabz # -################## +################# # Every package should define these variables $pkg_cmd_name = "crabz" $pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\crabz.exe" +$pkg_dst_bin = "$Env:USERPROFILE\.local\bin" $pkg_dst = "$pkg_dst_cmd" $pkg_src_cmd = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin\crabz.exe" @@ -15,47 +16,33 @@ $pkg_src_bin = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION\bin" $pkg_src_dir = "$Env:USERPROFILE\.local\opt\crabz-v$Env:WEBI_VERSION" $pkg_src = "$pkg_src_cmd" -New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | out-null +New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | Out-Null $pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE" # Fetch archive -IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) -{ - echo "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download" +IF (!(Test-Path -Path "$pkg_download")) { + Write-Output "Downloading crabz from $Env:WEBI_PKG_URL to $pkg_download" & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part" - & move "$pkg_download.part" "$pkg_download" + & Move-Item "$pkg_download.part" "$pkg_download" } -IF (!(Test-Path -Path "$pkg_src_cmd")) -{ - echo "Installing crabz" +IF (!(Test-Path -Path "$pkg_src_cmd")) { + Write-Output "Installing crabz" # TODO: create package-specific temp directory # Enter tmp - pushd .local\tmp + Push-Location .local\tmp - # Remove any leftover tmp cruft - Remove-Item -Path ".\crabz-v*" -Recurse -ErrorAction Ignore - Remove-Item -Path ".\crabz.exe" -Recurse -ErrorAction Ignore - - # NOTE: DELETE THIS COMMENT IF NOT USED - # Move single binary into root of temporary folder - #& move "$pkg_download" "crabz.exe" - - # Unpack archive file into this temporary directory - # Windows BSD-tar handles zip. Imagine that. - echo "Unpacking $pkg_download" - & tar xf "$pkg_download" - - # Settle unpacked archive into place - echo "Install Location: $pkg_src_cmd" - New-Item "$pkg_src_bin" -ItemType Directory -Force | out-null - Move-Item -Path ".\crabz-*\crabz.exe" -Destination "$pkg_src_bin" + # Settle unpacked archive into place + Write-Output "Install Location: $pkg_src_cmd" + New-Item "$pkg_src_bin" -ItemType Directory -Force | Out-Null + Copy-Item -Path "$pkg_download" -Destination "$pkg_src_cmd" # Exit tmp - popd + Pop-Location } -echo "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" -Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | out-null +Write-Output "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" +Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | Out-Null +New-Item "$pkg_dst_bin" -ItemType Directory -Force | Out-Null Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse diff --git a/crabz/install.sh b/crabz/install.sh index 907718b..6349bd2 100644 --- a/crabz/install.sh +++ b/crabz/install.sh @@ -1,46 +1,49 @@ #!/bin/sh -# shellcheck disable=SC2034 -# "'pkg_cmd_name' appears unused. Verify it or export it." +# The generic functions - version checks, download, extract, etc - are here: +# - https://github.com/webinstall/packages/branches/master/_webi/template.sh -__init_crabz() { - set -e - set -u +set -e +set -u - ################## - # Install crabz # - ################## +pkg_cmd_name="crabz" - # Every package should define these 6 variables - pkg_cmd_name="crabz" +# IMPORTANT: this let's other functions know to expect this to be a single file +WEBI_SINGLE=true - pkg_dst_cmd="$HOME/.local/bin/crabz" - pkg_dst="$pkg_dst_cmd" +# Every package should define these 6 variables +pkg_cmd_name="crabz" - pkg_src_cmd="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin/crabz" - pkg_src_dir="$HOME/.local/opt/crabz-v$WEBI_VERSION" - pkg_src="$pkg_src_cmd" +pkg_dst_cmd="$HOME/.local/bin/crabz" +#pkg_dst="$pkg_dst_cmd" - # pkg_install must be defined by every package - pkg_install() { - # ~/.local/opt/crabz-v0.99.9/bin - mkdir -p "$(dirname "${pkg_src_cmd}")" - - # mv ./crabz-*/crabz ~/.local/opt/crabz-v0.99.9/bin/crabz - mv ./crabz-* "${pkg_src_cmd}" - } - - # pkg_get_current_version is recommended, but not required - pkg_get_current_version() { - # 'crabz --version' has output in this format: - # crabz 0.99.9 (rev abcdef0123) - # This trims it down to just the version number: - # 0.99.9 - crabz --version 2> /dev/null | - head -n 1 | - cut -d ' ' -f 2 - } +pkg_src_cmd="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin/crabz" +#pkg_src_dir="$HOME/.local/opt/crabz-v$WEBI_VERSION/bin" +#pkg_src="$pkg_src_cmd" +pkg_get_current_version() { + # 'crabz version' has output in this format: + # crabz git:xxxxxxx + # Since that's not sortable, this prints v0.0.0 + # v0.0.0 + echo "v0.0.0" } -__init_crabz +pkg_install() { + # $HOME/.local/opt/crabz-v0.3.5/bin + mkdir -p "${pkg_src_bin}" + + # mv ./crabz* "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" + mv ./"${pkg_cmd_name}"* "${pkg_src_cmd}" + + # chmod a+x "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" + chmod a+x "${pkg_src_cmd}" +} + +pkg_link() { + # rm -f "$HOME/.local/bin/crabz" + rm -f "${pkg_dst_cmd}" + + # ln -s "$HOME/.local/opt/crabz-v0.3.5/bin/crabz" "$HOME/.local/bin/crabz" + ln -s "${pkg_src_cmd}" "${pkg_dst_cmd}" +} diff --git a/crabz/releases.js b/crabz/releases.js index 08a0c75..e93491e 100644 --- a/crabz/releases.js +++ b/crabz/releases.js @@ -4,10 +4,21 @@ var github = require('../_common/github.js'); var owner = 'sstadick'; var repo = 'crabz'; -module.exports = function (request) { - return github(request, owner, repo).then(function (all) { - return all; - }); +module.exports = async function (request) { + let all = await github(request, owner, repo); + + let releases = []; + for (let rel of all.releases) { + let isSrc = rel.download.includes('-src.'); + if (isSrc) { + continue; + } + + releases.push(rel); + } + all.releases = releases; + + return all; }; if (module === require.main) {