Skip to content

Commit

Permalink
Merge pull request #1037 from mcrowson/gzipped_project
Browse files Browse the repository at this point in the history 
gzippin
  • Loading branch information
Rich Jones authored Aug 14, 2017
2 parents 8a18eac + c466a0b commit ec39035
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 40 deletions.
9 changes: 5 additions & 4 deletions zappa/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,7 +689,7 @@ def deploy(self):
raise ClickException("Unable to upload handler to S3. Quitting.")

# Copy the project zip to the current project zip
current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
current_project_name = '{0!s}_current_project.tar.gz'.format(self.project_name)
success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
bucket_name=self.s3_bucket_name)
if not success: # pragma: no cover
Expand Down Expand Up @@ -840,7 +840,7 @@ def update(self):
raise ClickException("Unable to upload handler to S3. Quitting.")

# Copy the project zip to the current project zip
current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
current_project_name = '{0!s}_current_project.tar.gz'.format(self.project_name)
success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
bucket_name=self.s3_bucket_name)
if not success: # pragma: no cover
Expand Down Expand Up @@ -2016,7 +2016,8 @@ def create_package(self, output=None):
prefix=self.lambda_name,
use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True),
exclude=self.stage_config.get('exclude', []),
disable_progress=self.disable_progress
disable_progress=self.disable_progress,
archive_format='tarball'
)

# Make sure the normal venv is not included in the handler's zip
Expand Down Expand Up @@ -2168,7 +2169,7 @@ def create_package(self, output=None):

# If slim handler, path to project zip
if self.stage_config.get('slim_handler', False):
settings_s += "ZIP_PATH='s3://{0!s}/{1!s}_current_project.zip'\n".format(self.s3_bucket_name, self.project_name)
settings_s += "ARCHIVE_PATH='s3://{0!s}/{1!s}_current_project.tar.gz'\n".format(self.s3_bucket_name, self.project_name)

# since includes are for slim handler add the setting here by joining arbitrary list from zappa_settings file
# and tell the handler we are the slim_handler
Expand Down
77 changes: 52 additions & 25 deletions zappa/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,14 +387,19 @@ def create_lambda_zip( self,
include=None,
venv=None,
output=None,
disable_progress=False
disable_progress=False,
archive_format='zip'
):
"""
Create a Lambda-ready zip file of the current virtualenvironment and working directory.
Returns path to that file.
"""
# Validate archive_format
if archive_format not in ['zip', 'tarball']:
raise KeyError("The archive format to create a lambda package must be zip or tarball")

# Pip is a weird package.
# Calling this function in some environments without this can cause.. funkiness.
import pip
Expand All @@ -404,17 +409,20 @@ def create_lambda_zip( self,

cwd = os.getcwd()
if not output:
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
if archive_format == 'zip':
archive_fname = prefix + '-' + str(int(time.time())) + '.zip'
elif archive_format == 'tarball':
archive_fname = prefix + '-' + str(int(time.time())) + '.tar.gz'
else:
zip_fname = output
zip_path = os.path.join(cwd, zip_fname)
archive_fname = output
archive_path = os.path.join(cwd, archive_fname)

# Files that should be excluded from the zip
if exclude is None:
exclude = list()

# Exclude the zip itself
exclude.append(zip_path)
exclude.append(archive_path)

# Make sure that 'concurrent' is always forbidden.
# https://github.com/Miserlou/Zappa/issues/827
Expand Down Expand Up @@ -521,15 +529,20 @@ def splitpath(path):
print(e)
# XXX - What should we do here?

# Then zip it all up..
print("Packaging project as zip..")
try:
# import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError: # pragma: no cover
compression_method = zipfile.ZIP_STORED
# Then archive it all up..
if archive_format == 'zip':
print("Packaging project as zip.")

try:
compression_method = zipfile.ZIP_DEFLATED
except ImportError: # pragma: no cover
compression_method = zipfile.ZIP_STORED
archivef = zipfile.ZipFile(archive_path, 'w', compression_method)

elif archive_format == 'tarball':
print("Packaging project as gzipped tarball.")
archivef = tarfile.open(archive_path, 'w|gz')

zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):

for filename in files:
Expand Down Expand Up @@ -561,13 +574,23 @@ def splitpath(path):
# Related: https://github.com/Miserlou/Zappa/issues/682
os.chmod(os.path.join(root, filename), 0o755)

# Actually put the file into the proper place in the zip
# Related: https://github.com/Miserlou/Zappa/pull/716
zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
zipi.create_system = 3
zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
with open(os.path.join(root, filename), 'rb') as f:
zipf.writestr(zipi, f.read(), compression_method)
if archive_format == 'zip':
# Actually put the file into the proper place in the zip
# Related: https://github.com/Miserlou/Zappa/pull/716
zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
zipi.create_system = 3
zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
with open(os.path.join(root, filename), 'rb') as f:
archivef.writestr(zipi, f.read(), compression_method)
elif archive_format == 'tarball':
tarinfo = tarfile.TarInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
tarinfo.mode = 0o755

stat = os.stat(os.path.join(root, filename))
tarinfo.mtime = stat.st_mtime
tarinfo.size = stat.st_size
with open(os.path.join(root, filename), 'rb') as f:
archivef.addfile(tarinfo, f)

# Create python init file if it does not exist
# Only do that if there are sub folders or python files and does not conflict with a neighbouring module
Expand All @@ -580,12 +603,16 @@ def splitpath(path):
tmp_init = os.path.join(temp_project_path, '__init__.py')
open(tmp_init, 'a').close()
os.chmod(tmp_init, 0o755)
zipf.write(tmp_init,
os.path.join(root.replace(temp_project_path, ''),
os.path.join(root.replace(temp_project_path, ''), '__init__.py')))

arcname = os.path.join(root.replace(temp_project_path, ''),
os.path.join(root.replace(temp_project_path, ''), '__init__.py'))
if archive_format == 'zip':
archivef.write(tmp_init, arcname)
elif archive_format == 'tarball':
archivef.add(tmp_init, arcname)

# And, we're done!
zipf.close()
archivef.close()

# Trash the temp directory
shutil.rmtree(temp_project_path)
Expand All @@ -594,7 +621,7 @@ def splitpath(path):
# Remove the temporary handler venv folder
shutil.rmtree(venv)

return zip_fname
return archive_fname

def extract_lambda_package(self, package_name, path):
"""
Expand Down
21 changes: 10 additions & 11 deletions zappa/handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
import logging
import os
import sys
import tempfile
import traceback
import zipfile
import tarfile

from builtins import str
from werkzeug.wrappers import Response
Expand Down Expand Up @@ -97,9 +99,9 @@ def __init__(self, settings_name="zappa_settings", session=None):
os.environ[str(key)] = self.settings.ENVIRONMENT_VARIABLES[key]

# Pulling from S3 if given a zip path
project_zip_path = getattr(self.settings, 'ZIP_PATH', None)
if project_zip_path:
self.load_remote_project_zip(project_zip_path)
project_archive_path = getattr(self.settings, 'ARCHIVE_PATH', None)
if project_archive_path:
self.load_remote_project_archive(project_archive_path)


# Load compliled library to the PythonPath
Expand Down Expand Up @@ -145,7 +147,7 @@ def __init__(self, settings_name="zappa_settings", session=None):

self.wsgi_app = ZappaWSGIMiddleware(wsgi_app_function)

def load_remote_project_zip(self, project_zip_path):
def load_remote_project_archive(self, project_zip_path):
"""
Puts the project files from S3 in /tmp and adds to path
"""
Expand All @@ -157,16 +159,13 @@ def load_remote_project_zip(self, project_zip_path):
else:
boto_session = self.session

# Download the zip
# Download zip file from S3
remote_bucket, remote_file = parse_s3_url(project_zip_path)
s3 = boto_session.resource('s3')
archive_on_s3 = s3.Object(remote_bucket, remote_file).get()

zip_path = '/tmp/{0!s}'.format(remote_file)
s3.Object(remote_bucket, remote_file).download_file(zip_path)

# Unzip contents to project folder
with zipfile.ZipFile(zip_path, 'r') as z:
z.extractall(path=project_folder)
with tarfile.open(fileobj=archive_on_s3['Body'], mode="r|gz") as t:
t.extractall(project_folder)

# Add to project path
sys.path.insert(0, project_folder)
Expand Down

0 comments on commit ec39035

Please sign in to comment.