Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move stuff to contrib #1997

Merged
merged 2 commits into from
Feb 2, 2017
Merged

Move stuff to contrib #1997

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
96779e2
Move non-core like modules to luigi.contrib
Tarrasch Jan 24, 2017
1bef1d0
Make mrrunner module private to luigi
Tarrasch Jan 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions luigi/contrib/hadoop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@
import luigi.task
import luigi.contrib.gcs
import luigi.contrib.hdfs
import luigi.s3
from luigi import mrrunner
import luigi.contrib.s3
from luigi.contrib import mrrunner

if six.PY2:
from itertools import imap as map
Expand Down Expand Up @@ -460,7 +460,7 @@ def run_job(self, job, tracking_url_callback=None):
# atomic output: replace output with a temporary work directory
if self.end_job_with_atomic_move_dir:
illegal_targets = (
luigi.s3.S3FlagTarget, luigi.contrib.gcs.GCSFlagTarget)
luigi.contrib.s3.S3FlagTarget, luigi.contrib.gcs.GCSFlagTarget)
if isinstance(job.output(), illegal_targets):
raise TypeError("end_job_with_atomic_move_dir is not supported"
" for {}".format(illegal_targets))
Expand Down Expand Up @@ -533,7 +533,7 @@ def run_job(self, job, tracking_url_callback=None):

allowed_input_targets = (
luigi.contrib.hdfs.HdfsTarget,
luigi.s3.S3Target,
luigi.contrib.s3.S3Target,
luigi.contrib.gcs.GCSTarget)
for target in luigi.task.flatten(job.input_hadoop()):
if not isinstance(target, allowed_input_targets):
Expand All @@ -543,7 +543,7 @@ def run_job(self, job, tracking_url_callback=None):

allowed_output_targets = (
luigi.contrib.hdfs.HdfsTarget,
luigi.s3.S3FlagTarget,
luigi.contrib.s3.S3FlagTarget,
luigi.contrib.gcs.GCSFlagTarget)
if not isinstance(job.output(), allowed_output_targets):
raise TypeError('output must be one of: {}'.format(
Expand Down
4 changes: 4 additions & 0 deletions luigi/mrrunner.py → luigi/contrib/mrrunner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
#

"""
Since after Luigi 2.5.0, this is a private module to Luigi. Luigi users should
not rely on that importing this module works. Furthermore, "luigi mr streaming"
have been greatly superseeded by technoligies like Spark, Hive, etc.
The hadoop runner.
This module contains the main() method which will be used to run the
Expand Down
Loading