Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gzippin #1037

Merged
merged 3 commits into from
Aug 14, 2017
Merged

gzippin #1037

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2696d9b
gzippin
Aug 4, 2017
6551e34
perms
Aug 4, 2017
c466a0b
needed to add some tarinfo and open the gzip in streaming mode.
Aug 6, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions zappa/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,7 +689,7 @@ def deploy(self):
raise ClickException("Unable to upload handler to S3. Quitting.")

# Copy the project zip to the current project zip
current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
current_project_name = '{0!s}_current_project.tar.gz'.format(self.project_name)
success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
bucket_name=self.s3_bucket_name)
if not success: # pragma: no cover
Expand Down Expand Up @@ -840,7 +840,7 @@ def update(self):
raise ClickException("Unable to upload handler to S3. Quitting.")

# Copy the project zip to the current project zip
current_project_name = '{0!s}_current_project.zip'.format(self.project_name)
current_project_name = '{0!s}_current_project.tar.gz'.format(self.project_name)
success = self.zappa.copy_on_s3(src_file_name=self.zip_path, dst_file_name=current_project_name,
bucket_name=self.s3_bucket_name)
if not success: # pragma: no cover
Expand Down Expand Up @@ -2016,7 +2016,8 @@ def create_package(self, output=None):
prefix=self.lambda_name,
use_precompiled_packages=self.stage_config.get('use_precompiled_packages', True),
exclude=self.stage_config.get('exclude', []),
disable_progress=self.disable_progress
disable_progress=self.disable_progress,
archive_format='tarball'
)

# Make sure the normal venv is not included in the handler's zip
Expand Down Expand Up @@ -2168,7 +2169,7 @@ def create_package(self, output=None):

# If slim handler, path to project zip
if self.stage_config.get('slim_handler', False):
settings_s += "ZIP_PATH='s3://{0!s}/{1!s}_current_project.zip'\n".format(self.s3_bucket_name, self.project_name)
settings_s += "ARCHIVE_PATH='s3://{0!s}/{1!s}_current_project.tar.gz'\n".format(self.s3_bucket_name, self.project_name)

# since includes are for slim handler add the setting here by joining arbitrary list from zappa_settings file
# and tell the handler we are the slim_handler
Expand Down
77 changes: 52 additions & 25 deletions zappa/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,14 +387,19 @@ def create_lambda_zip( self,
include=None,
venv=None,
output=None,
disable_progress=False
disable_progress=False,
archive_format='zip'
):
"""
Create a Lambda-ready zip file of the current virtualenvironment and working directory.

Returns path to that file.

"""
# Validate archive_format
if archive_format not in ['zip', 'tarball']:
raise KeyError("The archive format to create a lambda package must be zip or tarball")

# Pip is a weird package.
# Calling this function in some environments without this can cause.. funkiness.
import pip
Expand All @@ -404,17 +409,20 @@ def create_lambda_zip( self,

cwd = os.getcwd()
if not output:
zip_fname = prefix + '-' + str(int(time.time())) + '.zip'
if archive_format == 'zip':
archive_fname = prefix + '-' + str(int(time.time())) + '.zip'
elif archive_format == 'tarball':
archive_fname = prefix + '-' + str(int(time.time())) + '.tar.gz'
else:
zip_fname = output
zip_path = os.path.join(cwd, zip_fname)
archive_fname = output
archive_path = os.path.join(cwd, archive_fname)

# Files that should be excluded from the zip
if exclude is None:
exclude = list()

# Exclude the zip itself
exclude.append(zip_path)
exclude.append(archive_path)

# Make sure that 'concurrent' is always forbidden.
# https://github.com/Miserlou/Zappa/issues/827
Expand Down Expand Up @@ -521,15 +529,20 @@ def splitpath(path):
print(e)
# XXX - What should we do here?

# Then zip it all up..
print("Packaging project as zip..")
try:
# import zlib
compression_method = zipfile.ZIP_DEFLATED
except ImportError: # pragma: no cover
compression_method = zipfile.ZIP_STORED
# Then archive it all up..
if archive_format == 'zip':
print("Packaging project as zip.")

try:
compression_method = zipfile.ZIP_DEFLATED
except ImportError: # pragma: no cover
compression_method = zipfile.ZIP_STORED
archivef = zipfile.ZipFile(archive_path, 'w', compression_method)

elif archive_format == 'tarball':
print("Packaging project as gzipped tarball.")
archivef = tarfile.open(archive_path, 'w|gz')

zipf = zipfile.ZipFile(zip_path, 'w', compression_method)
for root, dirs, files in os.walk(temp_project_path):

for filename in files:
Expand Down Expand Up @@ -561,13 +574,23 @@ def splitpath(path):
# Related: https://github.com/Miserlou/Zappa/issues/682
os.chmod(os.path.join(root, filename), 0o755)

# Actually put the file into the proper place in the zip
# Related: https://github.com/Miserlou/Zappa/pull/716
zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
zipi.create_system = 3
zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
with open(os.path.join(root, filename), 'rb') as f:
zipf.writestr(zipi, f.read(), compression_method)
if archive_format == 'zip':
# Actually put the file into the proper place in the zip
# Related: https://github.com/Miserlou/Zappa/pull/716
zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
zipi.create_system = 3
zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
with open(os.path.join(root, filename), 'rb') as f:
archivef.writestr(zipi, f.read(), compression_method)
elif archive_format == 'tarball':
tarinfo = tarfile.TarInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
tarinfo.mode = 0o755

stat = os.stat(os.path.join(root, filename))
tarinfo.mtime = stat.st_mtime
tarinfo.size = stat.st_size
with open(os.path.join(root, filename), 'rb') as f:
archivef.addfile(tarinfo, f)

# Create python init file if it does not exist
# Only do that if there are sub folders or python files and does not conflict with a neighbouring module
Expand All @@ -580,12 +603,16 @@ def splitpath(path):
tmp_init = os.path.join(temp_project_path, '__init__.py')
open(tmp_init, 'a').close()
os.chmod(tmp_init, 0o755)
zipf.write(tmp_init,
os.path.join(root.replace(temp_project_path, ''),
os.path.join(root.replace(temp_project_path, ''), '__init__.py')))

arcname = os.path.join(root.replace(temp_project_path, ''),
os.path.join(root.replace(temp_project_path, ''), '__init__.py'))
if archive_format == 'zip':
archivef.write(tmp_init, arcname)
elif archive_format == 'tarball':
archivef.add(tmp_init, arcname)

# And, we're done!
zipf.close()
archivef.close()

# Trash the temp directory
shutil.rmtree(temp_project_path)
Expand All @@ -594,7 +621,7 @@ def splitpath(path):
# Remove the temporary handler venv folder
shutil.rmtree(venv)

return zip_fname
return archive_fname

def extract_lambda_package(self, package_name, path):
"""
Expand Down
21 changes: 10 additions & 11 deletions zappa/handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,10 @@
import logging
import os
import sys
import tempfile
import traceback
import zipfile
import tarfile

from builtins import str
from werkzeug.wrappers import Response
Expand Down Expand Up @@ -97,9 +99,9 @@ def __init__(self, settings_name="zappa_settings", session=None):
os.environ[str(key)] = self.settings.ENVIRONMENT_VARIABLES[key]

# Pulling from S3 if given a zip path
project_zip_path = getattr(self.settings, 'ZIP_PATH', None)
if project_zip_path:
self.load_remote_project_zip(project_zip_path)
project_archive_path = getattr(self.settings, 'ARCHIVE_PATH', None)
if project_archive_path:
self.load_remote_project_archive(project_archive_path)


# Load compliled library to the PythonPath
Expand Down Expand Up @@ -145,7 +147,7 @@ def __init__(self, settings_name="zappa_settings", session=None):

self.wsgi_app = ZappaWSGIMiddleware(wsgi_app_function)

def load_remote_project_zip(self, project_zip_path):
def load_remote_project_archive(self, project_zip_path):
"""
Puts the project files from S3 in /tmp and adds to path
"""
Expand All @@ -157,16 +159,13 @@ def load_remote_project_zip(self, project_zip_path):
else:
boto_session = self.session

# Download the zip
# Download zip file from S3
remote_bucket, remote_file = parse_s3_url(project_zip_path)
s3 = boto_session.resource('s3')
archive_on_s3 = s3.Object(remote_bucket, remote_file).get()

zip_path = '/tmp/{0!s}'.format(remote_file)
s3.Object(remote_bucket, remote_file).download_file(zip_path)

# Unzip contents to project folder
with zipfile.ZipFile(zip_path, 'r') as z:
z.extractall(path=project_folder)
with tarfile.open(fileobj=archive_on_s3['Body'], mode="r|gz") as t:
t.extractall(project_folder)

# Add to project path
sys.path.insert(0, project_folder)
Expand Down