Add ability to disable data descriptor

fixes #18

lib/zstream.ex

@@ -47,6 +47,25 @@ defmodule Zstream do
 
 
     * `:mtime` (DateTime) - File last modication time. Defaults to system local time.
+
+  ## Disable Data Descriptor
+
+  By default, zstream will use the [data
+  descriptor](https://en.wikipedia.org/wiki/ZIP_(file_format)#Data_descriptor)
+  feature and skip the crc32 and size value in the local file
+  header. This might not work with some old versions of zip, since
+  this was a later addition to the zip format to support streaming.
+
+  If the file size and crc32 are known, this feature can be
+  disabled. NOTE: If `data_descriptor` is set to false, then the coder
+  should be set to `Zstream.Coder.Stored` as well.
+
+    * `:data_descriptor` (boolean) - Disable Data Descriptor. Defaults to true
+
+    * `:size` (integer) - Size of the File.
+
+    * `:crc32` (integer) - CRC32 of the File.
+
   """
   @spec entry(String.t(), Enumerable.t(), Keyword.t()) :: entry
   defdelegate entry(name, enum, options \\ []), to: Zstream.Zip

lib/zstream/protocol.ex

@@ -9,12 +9,21 @@ defmodule Zstream.Protocol do
 
   @comment "Created by Zstream"
 
-  def local_file_header(name, local_file_header_offset, options) do
+  def local_file_header(name, _local_file_header_offset, options) do
+    {crc32, c_size, size} =
+      if Keyword.fetch!(options, :data_descriptor) do
+        {0, 0, 0}
+      else
+        size = Keyword.fetch!(options, :size)
+        crc32 = Keyword.fetch!(options, :crc32)
+        {crc32, size, size}
+      end
+
     extra_field =
       zip64?(
         options,
         <<>>,
-        Extra.zip64_extended_info(0, 0, local_file_header_offset)
+        Extra.local_zip64_extended_info(size, c_size)
       )
 
     [
@@ -31,11 +40,11 @@ defmodule Zstream.Protocol do
         # last mod file date
         dos_date(Keyword.fetch!(options, :mtime))::little-size(16),
         # crc-32
-        0::little-size(32),
+        crc32::little-size(32),
         # compressed size
-        0::little-size(32),
+        zip64?(options, c_size, 0xFFFFFFFF)::little-size(32),
         # uncompressed size
-        0::little-size(32),
+        zip64?(options, size, 0xFFFFFFFF)::little-size(32),
         # file name length
         byte_size(name)::little-size(16),
         # extra field length
@@ -47,14 +56,19 @@ defmodule Zstream.Protocol do
   end
 
   def data_descriptor(crc32, compressed_size, uncompressed_size, options) do
-    if Keyword.fetch!(options, :zip64) do
-      # signature
-      <<0x08074B50::little-size(32), crc32::little-size(32), compressed_size::little-size(64),
-        uncompressed_size::little-size(64)>>
-    else
-      # signature
-      <<0x08074B50::little-size(32), crc32::little-size(32), compressed_size::little-size(32),
-        uncompressed_size::little-size(32)>>
+    cond do
+      !Keyword.fetch!(options, :data_descriptor) ->
+        []
+
+      Keyword.fetch!(options, :zip64) ->
+        # signature
+        <<0x08074B50::little-size(32), crc32::little-size(32), compressed_size::little-size(64),
+          uncompressed_size::little-size(64)>>
+
+      true ->
+        # signature
+        <<0x08074B50::little-size(32), crc32::little-size(32), compressed_size::little-size(32),
+          uncompressed_size::little-size(32)>>
     end
   end
 
@@ -179,10 +193,17 @@ defmodule Zstream.Protocol do
   defp general_purpose_bit_flag(options) do
     {encryption_coder, _opts} = Keyword.fetch!(options, :encryption_coder)
 
+    data_descriptor_flag =
+      if Keyword.fetch!(options, :data_descriptor) do
+        0x0008
+      else
+        0x0000
+      end
+
     # encryption bit set based on coder
     # bit 3 use data descriptor
     # bit 11 UTF-8 encoding of filename & comment fields
-    encryption_coder.general_purpose_flag() ||| 0x0008 ||| 0x0800
+    encryption_coder.general_purpose_flag() ||| data_descriptor_flag ||| 0x0800
   end
 
   defp external_file_attributes do

lib/zstream/zip.ex

@@ -29,7 +29,8 @@ defmodule Zstream.Zip do
 
   @default [
     coder: {Zstream.Coder.Deflate, []},
-    encryption_coder: {Zstream.EncryptionCoder.None, []}
+    encryption_coder: {Zstream.EncryptionCoder.None, []},
+    data_descriptor: true
   ]
 
   @global_default [

lib/zstream/zip/extra.ex

@@ -39,4 +39,9 @@ defmodule Zstream.Zip.Extra do
     <<0x0001::little-size(16), 28::little-size(16), size::little-size(64),
       c_size::little-size(64), offset::little-size(64), 0::little-size(32)>>
   end
+
+  def local_zip64_extended_info(size, c_size) do
+    <<0x0001::little-size(16), 16::little-size(16), size::little-size(64),
+      c_size::little-size(64)>>
+  end
 end

test/zstream_test.exs

@@ -31,6 +31,35 @@ defmodule ZstreamTest do
     ])
   end
 
+  test "zip with known size and crc" do
+    verify([
+      Zstream.entry("10.txt", ["123456789."],
+        coder: Zstream.Coder.Stored,
+        data_descriptor: false,
+        size: 10,
+        crc32: 3_692_204_934
+      )
+    ])
+
+    verify([
+      Zstream.entry("kafka_uncompressed", file("kafan.txt"),
+        coder: Zstream.Coder.Stored,
+        data_descriptor: false,
+        size: 33248,
+        crc32: 2_503_591_999
+      )
+    ])
+
+    verify([
+      Zstream.entry("empty_file_1", [],
+        coder: Zstream.Coder.Stored,
+        data_descriptor: false,
+        size: 0,
+        crc32: 0
+      )
+    ])
+  end
+
   test "unzip" do
     verify_unzip("docx")
     verify_unzip("uncompressed")
@@ -271,6 +300,12 @@ defmodule ZstreamTest do
     Logger.debug(response)
     assert exit_code == 0
 
+    if Keyword.get(options, :debug) do
+      {response, exit_code} = System.cmd("zipdetails", [path])
+      Logger.debug(response)
+      assert exit_code == 0
+    end
+
     {response, exit_code} = System.cmd("unzip", ["-t", path])
     Logger.debug(response)
     assert exit_code == 0