diff --git a/.travis.yml b/.travis.yml index 8b5e836cb3..e39294141a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - GCS_TEST_BUCKET=luigi-travistestenvironment - GOOGLE_APPLICATION_CREDENTIALS=test/gcloud-credentials.json matrix: - - TOXENV=pep8 + - TOXENV=flake8 - TOXENV=docs - TOXENV=py27-nonhdfs - TOXENV=py33-nonhdfs diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 061d00deea..69942f1072 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -12,7 +12,7 @@ then run one of the tox commands below. # These commands are pretty fast and will tell if you if you've # broken something major: - tox -e pep8 + tox -e flake8 tox -e py27-nonhdfs # You can also test particular files for even faster iterations @@ -22,7 +22,7 @@ then run one of the tox commands below. tox -e py33-cdh tox -e py34-hdp -Where ``pep8`` is the lint checking, ``py27`` is obviously Python 2.7. +Where ``flake8`` is the lint checking, ``py27`` is obviously Python 2.7. ``nonhdfs`` are tests not running on the Hadoop minicluster and ``cdh`` and ``hdp`` are two different hadoop distributions. For most local development it's usually enough to run the lint checking and a python version for ``nonhdfs`` diff --git a/doc/conf.py b/doc/conf.py index 418520c802..6e5fbac330 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -209,22 +209,22 @@ # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'Luigi.tex', u'Luigi Documentation', - u'Erik Bernhardsson and Elias Freider', 'manual'), + ('index', 'Luigi.tex', u'Luigi Documentation', + u'Erik Bernhardsson and Elias Freider', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -267,9 +267,9 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Luigi', u'Luigi Documentation', - u'Erik Bernhardsson and Elias Freider', 'Luigi', 'One line description of project.', - 'Miscellaneous'), + ('index', 'Luigi', u'Luigi Documentation', + u'Erik Bernhardsson and Elias Freider', 'Luigi', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/doc/settings.py b/doc/settings.py deleted file mode 100644 index cf1645f930..0000000000 --- a/doc/settings.py +++ /dev/null @@ -1 +0,0 @@ -STYLE_CACHE_DIRECTORY=/tmp diff --git a/examples/execution_summary_example.py b/examples/execution_summary_example.py index bbaba74e21..00e5901033 100644 --- a/examples/execution_summary_example.py +++ b/examples/execution_summary_example.py @@ -47,9 +47,6 @@ from __future__ import print_function -import os -import shutil -import time import datetime import luigi diff --git a/luigi/__init__.py b/luigi/__init__.py index 09d1abb184..f6e3d7e2ef 100644 --- a/luigi/__init__.py +++ b/luigi/__init__.py @@ -48,3 +48,14 @@ from luigi.event import Event from .tools import range # just makes the tool classes available from command line + + +__all__ = [ + 'task', 'Task', 'Config', 'ExternalTask', 'WrapperTask', 'namespace', + 'target', 'Target', 'File', 'LocalTarget', 'rpc', 'RemoteScheduler', + 'RPCError', 'parameter', 'Parameter', 'DateParameter', 'MonthParameter', + 'YearParameter', 'DateHourParameter', 'DateMinuteParameter', 'range', + 'DateIntervalParameter', 'TimeDeltaParameter', 'IntParameter', + 'FloatParameter', 'BooleanParameter', 'BoolParameter', 'TaskParameter', + 'configuration', 'interface', 'file', 'run', 'build', 'event', 'Event' +] diff --git a/luigi/contrib/hadoop.py b/luigi/contrib/hadoop.py index 4f7e268168..49966d6dd2 100644 --- a/luigi/contrib/hadoop.py +++ b/luigi/contrib/hadoop.py @@ -304,7 +304,7 @@ def track_process(arglist, tracking_url_callback, env=None): hadoop_context.job_id = job_id # Read the rest + stdout - err = ''.join(err_lines + [err_line for err_line in proc.stderr]) + err = ''.join(err_lines + [an_err_line for an_err_line in proc.stderr]) temp_stdout.seek(0) out = ''.join(temp_stdout.readlines()) diff --git a/luigi/contrib/hive.py b/luigi/contrib/hive.py index f345ae874e..92af568dec 100644 --- a/luigi/contrib/hive.py +++ b/luigi/contrib/hive.py @@ -220,7 +220,6 @@ class HiveThriftContext(object): def __enter__(self): try: - from thrift import Thrift from thrift.transport import TSocket from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol diff --git a/luigi/contrib/pig.py b/luigi/contrib/pig.py index 198da65241..8f657067dd 100644 --- a/luigi/contrib/pig.py +++ b/luigi/contrib/pig.py @@ -148,7 +148,7 @@ def track_and_progress(self, cmd): logger.info(t) # Read the rest + stdout - err = ''.join(err_lines + [err_line.decode('utf8') for err_line in proc.stderr]) + err = ''.join(err_lines + [an_err_line.decode('utf8') for an_err_line in proc.stderr]) if proc.returncode == 0: logger.info("Job completed successfully!") else: diff --git a/luigi/contrib/webhdfs.py b/luigi/contrib/webhdfs.py index 6c43bc05df..7bfcb84a32 100644 --- a/luigi/contrib/webhdfs.py +++ b/luigi/contrib/webhdfs.py @@ -23,7 +23,6 @@ import logging import os -import sys from luigi import six diff --git a/luigi/execution_summary.py b/luigi/execution_summary.py index 30ba54af26..5b1544bc5c 100644 --- a/luigi/execution_summary.py +++ b/luigi/execution_summary.py @@ -22,7 +22,6 @@ """ import textwrap -import datetime def _partition_tasks(worker): @@ -309,7 +308,6 @@ def _summary_format(set_tasks, worker): group_tasks = {} for status, task_dict in set_tasks.items(): group_tasks[status] = _group_tasks_by_name_and_status(task_dict) - str_tasks = {} comments = _get_comments(group_tasks) num_all_tasks = len(set_tasks["already_done"]) + len(set_tasks["completed"]) + len(set_tasks["failed"]) + len(set_tasks["still_pending_ext"]) + len(set_tasks["still_pending_not_ext"]) str_output = '' diff --git a/luigi/hadoop.py b/luigi/hadoop.py index b8e873a15f..d183eafbc3 100644 --- a/luigi/hadoop.py +++ b/luigi/hadoop.py @@ -21,6 +21,6 @@ import warnings -from luigi.contrib.hadoop import * +from luigi.contrib.hadoop import * # NOQA warnings.warn("luigi.hadoop module has been moved to luigi.contrib.hadoop", DeprecationWarning) diff --git a/luigi/hadoop_jar.py b/luigi/hadoop_jar.py index 42aea30dbf..a54ded0c8a 100644 --- a/luigi/hadoop_jar.py +++ b/luigi/hadoop_jar.py @@ -21,7 +21,7 @@ import warnings -from luigi.contrib.hadoop_jar import * +from luigi.contrib.hadoop_jar import * # NOQA warnings.warn("luigi.hadoop_jar module has been moved to luigi.contrib.hadoop_jar", DeprecationWarning) diff --git a/luigi/hdfs.py b/luigi/hdfs.py index 827c346c38..e3af89fd21 100644 --- a/luigi/hdfs.py +++ b/luigi/hdfs.py @@ -21,7 +21,7 @@ import warnings -from luigi.contrib.hdfs import * +from luigi.contrib.hdfs import * # NOQA warnings.warn("luigi.hdfs module has been moved to luigi.contrib.hdfs", DeprecationWarning) diff --git a/luigi/hive.py b/luigi/hive.py index c3138637bc..0ff98aa193 100644 --- a/luigi/hive.py +++ b/luigi/hive.py @@ -21,7 +21,7 @@ import warnings -from luigi.contrib.hive import * +from luigi.contrib.hive import * # NOQA warnings.warn("luigi.hive module has been moved to luigi.contrib.hive", DeprecationWarning) diff --git a/luigi/interface.py b/luigi/interface.py index d4937350aa..1317b6b5cb 100644 --- a/luigi/interface.py +++ b/luigi/interface.py @@ -238,8 +238,6 @@ def parse_task(self, cmdline_args=None): _add_global_parameters(parser) - task_names = Register.task_names() - # Parse global arguments and pull out the task name. # We used to do this using subparsers+command, but some issues with # argparse across different versions of Python (2.7.9) made it hard. diff --git a/luigi/rpc.py b/luigi/rpc.py index f06fb96a79..267322f1f4 100644 --- a/luigi/rpc.py +++ b/luigi/rpc.py @@ -25,7 +25,7 @@ import socket import time -from luigi.six.moves.urllib.parse import urljoin, urlencode, ParseResult +from luigi.six.moves.urllib.parse import urljoin, urlencode from luigi.six.moves.urllib.request import urlopen from luigi.six.moves.urllib.error import URLError diff --git a/luigi/scalding.py b/luigi/scalding.py index b35ce68920..1f03406aeb 100644 --- a/luigi/scalding.py +++ b/luigi/scalding.py @@ -20,6 +20,6 @@ import warnings -from luigi.contrib.scalding import * +from luigi.contrib.scalding import * # NOQA warnings.warn("luigi.scalding has now moved to luigi.contrib.scalding", DeprecationWarning, stacklevel=3) diff --git a/luigi/scheduler.py b/luigi/scheduler.py index ca9c6a5866..692309a867 100644 --- a/luigi/scheduler.py +++ b/luigi/scheduler.py @@ -825,7 +825,7 @@ def _upstream_status(self, task_id, upstream_status_table): elif upstream_status_table[dep_id] == '' and dep.deps: # This is the postorder update step when we set the # status based on the previously calculated child elements - upstream_status = [upstream_status_table.get(task_id, '') for task_id in dep.deps] + upstream_status = [upstream_status_table.get(a_task_id, '') for a_task_id in dep.deps] upstream_status.append('') # to handle empty list status = max(upstream_status, key=UPSTREAM_SEVERITY_KEY) upstream_status_table[dep_id] = status diff --git a/luigi/server.py b/luigi/server.py index e6f47f1098..2b33c2bf9a 100644 --- a/luigi/server.py +++ b/luigi/server.py @@ -53,7 +53,6 @@ import tornado.netutil import tornado.web -from luigi import configuration from luigi.scheduler import CentralPlannerScheduler diff --git a/luigi/task.py b/luigi/task.py index c8fb83cb71..b092a6be44 100644 --- a/luigi/task.py +++ b/luigi/task.py @@ -31,7 +31,7 @@ from luigi import six from luigi import parameter -from luigi.task_register import Register, TaskClassException +from luigi.task_register import Register Parameter = parameter.Parameter logger = logging.getLogger('luigi-interface') diff --git a/luigi/util.py b/luigi/util.py index 892e2ad7b4..4ebc1532bb 100644 --- a/luigi/util.py +++ b/luigi/util.py @@ -23,7 +23,7 @@ from luigi import task from luigi import parameter -from luigi.deprecate_kwarg import deprecate_kwarg # removing this breaks code +from luigi.deprecate_kwarg import deprecate_kwarg # NOQA: removing this breaks code if six.PY3: xrange = range diff --git a/luigi/webhdfs.py b/luigi/webhdfs.py index bc826511ca..41a8906083 100644 --- a/luigi/webhdfs.py +++ b/luigi/webhdfs.py @@ -20,7 +20,7 @@ import warnings -from luigi.contrib.webhdfs import * +from luigi.contrib.webhdfs import * # NOQA warnings.warn("luigi.webhdfs module has been moved to luigi.contrib.webhdfs", DeprecationWarning) diff --git a/luigi/worker.py b/luigi/worker.py index 55e8ddcfd9..c12935fd74 100644 --- a/luigi/worker.py +++ b/luigi/worker.py @@ -49,7 +49,8 @@ from luigi.task_register import load_task from luigi.scheduler import DISABLED, DONE, FAILED, PENDING, RUNNING, SUSPENDED, CentralPlannerScheduler from luigi.target import Target -from luigi.task import Task, flatten, getpaths, TaskClassException, Config +from luigi.task import Task, flatten, getpaths, Config +from luigi.task_register import TaskClassException from luigi.parameter import FloatParameter, IntParameter, BoolParameter try: diff --git a/setup.py b/setup.py index a5ea62e11e..c02846cd3b 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ # the License. import os -import sys try: from setuptools import setup @@ -22,7 +21,7 @@ def get_static_files(path): - return [os.path.join(dirpath.replace("luigi/", ""), ext) + return [os.path.join(dirpath.replace("luigi/", ""), ext) for (dirpath, dirnames, filenames) in os.walk(path) for ext in ["*.html", "*.js", "*.css", "*.png"]] diff --git a/test/cmdline_test.py b/test/cmdline_test.py index 4f629ce873..842ffd939f 100644 --- a/test/cmdline_test.py +++ b/test/cmdline_test.py @@ -49,7 +49,7 @@ class AmbiguousClass(luigi.Task): pass -class AmbiguousClass(luigi.Task): +class AmbiguousClass(luigi.Task): # NOQA pass @@ -59,7 +59,7 @@ def run(self): self.x = 42 -class TaskWithSameName(luigi.Task): +class TaskWithSameName(luigi.Task): # NOQA # there should be no ambiguity def run(self): diff --git a/test/contrib/hadoop_test.py b/test/contrib/hadoop_test.py index 8070d597d5..eb0ba6cdc1 100644 --- a/test/contrib/hadoop_test.py +++ b/test/contrib/hadoop_test.py @@ -381,7 +381,3 @@ def test_create_packages_archive_package_subpackage_submodule(self, tar): package_subpackage_submodule = __import__("package.subpackage.submodule", None, None, 'dummy') luigi.contrib.hadoop.create_packages_archive([package_subpackage_submodule], '/dev/null') self._assert_package_subpackage(tar.return_value.add) - - -if __name__ == '__main__': - HadoopJobTest.test_run_real() diff --git a/test/contrib/sqla_test.py b/test/contrib/sqla_test.py index fa02c92541..f45ffc22d7 100644 --- a/test/contrib/sqla_test.py +++ b/test/contrib/sqla_test.py @@ -305,7 +305,7 @@ def output(self): def run(self): out = self.output().open("w") - for task in TASK_LIST: + for task in self.TASK_LIST: out.write("dummy_" + task) out.close() @@ -337,7 +337,7 @@ def copy(self, conn, ins_rows, table_bound): conn.execute(ins, ins_rows) def rows(self): - for task in TASK_LIST: + for task in self.TASK_LIST: yield task.strip("\n").split("\t") # Running only task1, and task2 should fail diff --git a/test/parameter_test.py b/test/parameter_test.py index f3f4fdf2bf..b4ed4de3c3 100644 --- a/test/parameter_test.py +++ b/test/parameter_test.py @@ -16,14 +16,13 @@ # import datetime -from helpers import unittest, LuigiTestCase +from helpers import unittest, with_config, LuigiTestCase from datetime import timedelta import luigi import luigi.date_interval import luigi.interface import luigi.notifications -from helpers import with_config, LuigiTestCase from luigi.mock import MockTarget, MockFileSystem from luigi.parameter import ParameterException from worker_test import email_patch diff --git a/test/redshift_test.py b/test/redshift_test.py index bbff18087d..8af3830afe 100644 --- a/test/redshift_test.py +++ b/test/redshift_test.py @@ -31,7 +31,7 @@ if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3): # spulec/moto#308 - mock_s3 = unittest.skip('moto mock doesn\'t work with python3.4') + mock_s3 = unittest.skip('moto mock doesn\'t work with python3.4') # NOQA luigi.notifications.DEBUG = True diff --git a/test/server_test.py b/test/server_test.py index 3550a17963..c7dca44038 100644 --- a/test/server_test.py +++ b/test/server_test.py @@ -99,7 +99,7 @@ def __init__(self): self.unix_socket = os.path.join(self.temp_dir, 'luigid.sock') def run_server(self): - luigi.server.run(unix_socket=unix_socket) + luigi.server.run(unix_socket=self.unix_socket) def scheduler(self): url = ParseResult( diff --git a/test/worker_test.py b/test/worker_test.py index b960030d57..c321c0a0e5 100644 --- a/test/worker_test.py +++ b/test/worker_test.py @@ -24,12 +24,11 @@ import tempfile import threading import time -from helpers import unittest, skipOnTravis +from helpers import unittest, with_config, skipOnTravis import luigi.notifications import luigi.worker import mock -from helpers import with_config, skipOnTravis from luigi import ExternalTask, RemoteScheduler, Task from luigi.mock import MockTarget, MockFileSystem from luigi.scheduler import CentralPlannerScheduler diff --git a/tox.ini b/tox.ini index 13052dd955..82e1469243 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,33,34}-{cdh,hdp,nonhdfs,gcloud,postgres,unixsockets}, pypy-scheduler, docs, pep8 +envlist = py{27,33,34}-{cdh,hdp,nonhdfs,gcloud,postgres,unixsockets}, pypy-scheduler, docs, flake8 skipsdist = True [testenv] @@ -51,9 +51,11 @@ commands = coverage combine coveralls -[testenv:pep8] -deps = pep8 -commands = pep8 --exclude=luigi/six.py --ignore E501,E402,E731 luigi test examples bin +[testenv:flake8] +deps = flake8 +commands = flake8 --max-line-length=384 --exclude=doc,test,luigi/six.py + flake8 --max-line-length=100 --ignore=E265 doc + flake8 --max-line-length=252 --ignore=F401,F841 test [testenv:autopep8] deps = autopep8