Compress help text with gzip.

Config.in

@@ -106,6 +106,14 @@ config TOYBOX_HELP_DASHDASH
 	  optstring. (Use TOYFLAG_NOHELP to disable.) Produces the same output
 	  as "help command". --version shows toybox version.
 
+config TOYBOX_ZHELP
+	bool "compress help text"
+	default y
+	depends on TOYBOX_HELP
+	help
+	  Compress help with gzip -9, deflating when displayed. This makes the
+	  binary smaller but can increase runtime memory usage.
+
 config TOYBOX_FREE
 	bool "Free memory unnecessarily"
 	default n

lib/deflate.c

@@ -26,14 +26,14 @@ struct deflate {
 
   // Compressed data buffer (extra space malloced at end)
   unsigned pos, len;
-  int infd, outfd;
-  char data[];
+  int infd, outfd, outbuflen;
+  char *outbuf, data[];
 };
 
 // little endian bit buffer
 struct bitbuf {
   int fd, bitpos, len, max;
-  char buf[];
+  char *buf, data[];
 };
 
 // malloc a struct bitbuf
@@ -43,6 +43,7 @@ static struct bitbuf *bitbuf_init(int fd, int size)
 
   bb->max = size;
   bb->fd = fd;
+  bb->buf = bb->data;
 
   return bb;
 }
@@ -57,7 +58,7 @@ static int bitbuf_skip(struct bitbuf *bb, int bits)
   while (pos >= (len = bb->len<<3)) {
     pos -= len;
     if (1 > (bb->len = read(bb->fd, bb->buf, bb->max))) {
-      if (!bb->len && !bits) break;
+      if (!bits) break;
       error_exit("inflate EOF");
     }
   }
@@ -135,16 +136,26 @@ static void bitbuf_put(struct bitbuf *bb, int data, int len)
   }
 }
 
+// Output inflated data
+static void inflate_out(struct deflate *dd, int len)
+{
+  if (!len) return;
+  if (dd->outfd!=-1) xwrite(dd->outfd, dd->data, len);
+  else if (len>dd->outbuflen) error_exit("inflate too big");
+  else {
+    memcpy(dd->outbuf, dd->data, len);
+    dd->outbuf += len;
+    dd->outbuflen -= len;
+  }
+  if (dd->crcfunc) dd->crcfunc(dd, dd->data, len);
+}
+
 static void output_byte(struct deflate *dd, char sym)
 {
   int pos = dd->pos++ & 32767;
 
   dd->data[pos] = sym;
-
-  if (pos == 32767) {
-    xwrite(dd->outfd, dd->data, 32768);
-    if (dd->crcfunc) dd->crcfunc(dd, dd->data, 32768);
-  }
+  if (pos == 32767) inflate_out(dd, 32768);
 }
 
 // Huffman coding uses bits to traverse a binary tree to a leaf node,
@@ -313,10 +324,7 @@ static void inflate(struct deflate *dd, struct bitbuf *bb)
     if (final) break;
   }
 
-  if (dd->pos & 32767) {
-    xwrite(dd->outfd, dd->data, dd->pos&32767);
-    if (dd->crcfunc) dd->crcfunc(dd, dd->data, dd->pos&32767);
-  }
+  if (dd->pos & 32767) inflate_out(dd, dd->pos&32767);
 }
 
 // Deflate from dd->infd to bitbuf
@@ -337,6 +345,7 @@ static void deflate(struct deflate *dd, struct bitbuf *bb)
     // dd->len += len;  crcfunc advances len TODO
 
     // store block as literal
+// TODO: actually compress!
     bitbuf_put(bb, final, 1);
     bitbuf_put(bb, 0, 1);
 
@@ -481,28 +490,23 @@ long long gzip_fd(int infd, int outfd)
   return rc;
 }
 
-long long gunzip_fd(int infd, int outfd)
+long long gunzip_common(struct bitbuf *bb, struct deflate *dd)
 {
-  struct bitbuf *bb = bitbuf_init(infd, 4096);
-  struct deflate *dd = init_deflate(0);
   long long rc = 0;
 
   // Little endian crc table
   crc_init(dd->crctable, 1);
   dd->crcfunc = gzip_crc;
-  dd->outfd = outfd;
 
   do {
     if (!is_gzip(bb)) error_exit("not gzip");
 
     inflate(dd, bb);
-
     // tail: crc32, len32
     bitbuf_skip(bb, (8-bb->bitpos)&7);
     if (~dd->crc != bitbuf_get(bb, 32) || dd->len != bitbuf_get(bb, 32))
       error_exit("bad crc");
     rc += dd->len;
-
     bitbuf_skip(bb, (8-bb->bitpos)&7);
     dd->pos = dd->len = 0;
   } while (bitbuf_skip(bb, 0));
@@ -511,3 +515,26 @@ long long gunzip_fd(int infd, int outfd)
 
   return rc;
 }
+
+long long gunzip_mem(char *inbuf, int inlen, char *outbuf, int outlen)
+{
+  struct bitbuf *bb = bitbuf_init(-1, 0);
+  struct deflate *dd = init_deflate(0);
+
+  bb->buf = inbuf;
+  bb->max = bb->len = inlen;
+  dd->outfd = -1;
+  dd->outbuf = outbuf;
+  dd->outbuflen = outlen;
+
+  return gunzip_common(bb, dd);
+}
+
+long long gunzip_fd(int infd, int outfd)
+{
+  struct bitbuf *bb = bitbuf_init(infd, 4096);
+  struct deflate *dd = init_deflate(0);
+
+  dd->outfd = outfd;
+  return gunzip_common(bb, dd);
+}

lib/lib.h

@@ -383,8 +383,7 @@ int comma_remove(char *optlist, char *opt);
 
 long long gzip_fd(int infd, int outfd);
 long long gunzip_fd(int infd, int outfd);
-long long gunzip_fd_preload(int infd, int outfd, char *buf, unsigned len);
-
+long long gunzip_mem(char *inbuf, int inlen, char *outbuf, int outlen);
 
 // getmountlist.c
 struct mtab_list {

main.c

@@ -76,38 +76,50 @@ static const char help_data[] =
 #include "generated/newtoys.h"
 ;
 
+#if CFG_TOYBOX_ZHELP
+#include "generated/zhelp.h"
+#else
+static char *zhelp_data = 0;
+#define ZHELP_LEN 0
+#endif
+
 void show_help(FILE *out, int flags)
 {
   int i = toys.which-toy_list;
-  char *s, *ss;
-
-  if (CFG_TOYBOX_HELP) {
-    if (flags & HELP_HEADER)
-      fprintf(out, "Toybox %s"USE_TOYBOX(" multicall binary")"%s\n\n",
-        toybox_version, (CFG_TOYBOX && i) ? " (see toybox --help)"
-        : " (see https://landley.net/toybox)");
-
-    for (;;) {
-      s = (void *)help_data;
-      while (i--) s += strlen(s) + 1;
-      // If it's an alias, restart search for real name
-      if (*s != 255) break;
-      i = toy_find(++s)-toy_list;
-      if ((flags & HELP_SEE) && toy_list[i].flags) {
-        if (flags & HELP_HTML) fprintf(out, "See <a href=#%s>%s</a>\n", s, s);
-        else fprintf(out, "%s see %s\n", toys.which->name, s);
-
-        return;
-      }
-    }
+  char *s, *ss, *hd;
+
+  if (!CFG_TOYBOX_HELP) return;
 
-    if (!(flags & HELP_USAGE)) fprintf(out, "%s\n", s);
-    else {
-      strstart(&s, "usage: ");
-      for (ss = s; *ss && *ss!='\n'; ss++);
-      fprintf(out, "%.*s\n", (int)(ss-s), s);
+  if (CFG_TOYBOX_ZHELP)
+    gunzip_mem(zhelp_data, sizeof(zhelp_data), hd = xmalloc(ZHELP_LEN),
+      ZHELP_LEN);
+  else hd = (void *)help_data;
+
+  if (flags & HELP_HEADER)
+    fprintf(out, "Toybox %s"USE_TOYBOX(" multicall binary")"%s\n\n",
+      toybox_version, (CFG_TOYBOX && i) ? " (see toybox --help)"
+      : " (see https://landley.net/toybox)");
+
+  for (;;) {
+    s = (void *)help_data;
+    while (i--) s += strlen(s) + 1;
+    // If it's an alias, restart search for real name
+    if (*s != 255) break;
+    i = toy_find(++s)-toy_list;
+    if ((flags & HELP_SEE) && toy_list[i].flags) {
+      if (flags & HELP_HTML) fprintf(out, "See <a href=#%s>%s</a>\n", s, s);
+      else fprintf(out, "%s see %s\n", toys.which->name, s);
+
+      return;
     }
   }
+
+  if (!(flags & HELP_USAGE)) fprintf(out, "%s\n", s);
+  else {
+    strstart(&s, "usage: ");
+    for (ss = s; *ss && *ss!='\n'; ss++);
+    fprintf(out, "%.*s\n", (int)(ss-s), s);
+  }
 }
 
 static void unknown(char *name)

scripts/install.c

@@ -4,6 +4,9 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
 #include "generated/config.h"
 #include "lib/toyflags.h"
 
@@ -16,11 +19,28 @@ struct {char *name; int flags;} toy_list[] = {
 #include "generated/newtoys.h"
 };
 
+#undef NEWTOY
+#undef OLDTOY
+#define NEWTOY(name,opt,flags) HELP_##name "\0"
+#if CFG_TOYBOX
+#define OLDTOY(name,oldname,flags) "\xff" #oldname "\0"
+#else
+#define OLDTOY(name, oldname, flags) HELP_##oldname "\0"
+#endif
+
+#include "generated/help.h"
+static char help_data[] =
+#include "generated/newtoys.h"
+;
+
 int main(int argc, char *argv[])
 {
   static char *toy_paths[]={"usr/","bin/","sbin/",0};
   int i, len = 0;
 
+  if (argc>1 && !strcmp(argv[1], "--help"))
+    exit(sizeof(help_data)!=write(1, help_data, sizeof(help_data)));
+
   // Output list of applets.
   for (i=1; i<sizeof(toy_list)/sizeof(*toy_list); i++) {
     int fl = toy_list[i].flags;

scripts/make.sh

@@ -158,7 +158,6 @@ then
   # happy. New ones have '\n' so can replace one line with two without all
   # the branches and tedious mucking about with hyperspace.
   # TODO: clean this up to use modern stuff.
-
   $SED -n \
     -e 's/^# CONFIG_\(.*\) is not set.*/\1/' \
     -e 't notset' \
@@ -259,13 +258,25 @@ then
     toys/*/*.c lib/*.c | "$UNSTRIPPED"/mktags > "$GENDIR"/tags.h
 fi
 
+# Create help.h, and zhelp.h if zcat enabled
 hostcomp config2help
 if isnewer help.h "$GENDIR"/Config.in
 then
-  "$UNSTRIPPED"/config2help Config.in $KCONFIG_CONFIG > "$GENDIR"/help.h || exit 1
+  "$UNSTRIPPED"/config2help Config.in $KCONFIG_CONFIG > "$GENDIR"/help.h||exit 1
 fi
-[ -z "$DIDNEWER" ] || echo }
 
+if grep -qx 'CONFIG_TOYBOX_ZHELP=y' "$KCONFIG_CONFIG"
+then
+  do_loudly $HOSTCC -I . scripts/install.c -o "$UNSTRIPPED"/instlist || exit 1
+  { echo "#define ZHELP_LEN $("$UNSTRIPPED"/instlist --help | wc -c)" &&
+    "$UNSTRIPPED"/instlist --help | gzip -9 | od -Anone -vtx1 | \
+    sed 's/ /,0x/g;1s/^,/static char zhelp_data[] = {\n /;$s/.*/&};/'
+  } > "$GENDIR"/zhelp.h || exit 1
+else
+  rm -f "$GENDIR"/zhelp.h
+fi
+
+[ -z "$DIDNEWER" ] || echo }
 [ -n "$NOBUILD" ] && exit 0
 
 echo "Compile $OUTNAME"