Make luigi package flake8 compliant

.travis.yml

@@ -14,7 +14,7 @@ env:
     - GCS_TEST_BUCKET=luigi-travistestenvironment
     - GOOGLE_APPLICATION_CREDENTIALS=test/gcloud-credentials.json
   matrix:
-    - TOXENV=pep8
+    - TOXENV=flake8
     - TOXENV=docs
     - TOXENV=py27-nonhdfs
     - TOXENV=py33-nonhdfs

CONTRIBUTING.rst

@@ -12,7 +12,7 @@ then run one of the tox commands below.
 
     # These commands are pretty fast and will tell if you if you've
     # broken something major:
-    tox -e pep8
+    tox -e flake8
     tox -e py27-nonhdfs
     
     # You can also test particular files for even faster iterations
@@ -22,7 +22,7 @@ then run one of the tox commands below.
     tox -e py33-cdh
     tox -e py34-hdp
 
-Where ``pep8`` is the lint checking, ``py27`` is obviously Python 2.7.
+Where ``flake8`` is the lint checking, ``py27`` is obviously Python 2.7.
 ``nonhdfs`` are tests not running on the Hadoop minicluster and ``cdh`` and
 ``hdp`` are two different hadoop distributions. For most local development it's
 usually enough to run the lint checking and a python version for ``nonhdfs``

doc/conf.py

@@ -209,22 +209,22 @@
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
 
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
 
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  ('index', 'Luigi.tex', u'Luigi Documentation',
-   u'Erik Bernhardsson and Elias Freider', 'manual'),
+    ('index', 'Luigi.tex', u'Luigi Documentation',
+     u'Erik Bernhardsson and Elias Freider', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -267,9 +267,9 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Luigi', u'Luigi Documentation',
-   u'Erik Bernhardsson and Elias Freider', 'Luigi', 'One line description of project.',
-   'Miscellaneous'),
+    ('index', 'Luigi', u'Luigi Documentation',
+     u'Erik Bernhardsson and Elias Freider', 'Luigi', 'One line description of project.',
+     'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.

examples/execution_summary_example.py

@@ -47,9 +47,6 @@
 
 
 from __future__ import print_function
-import os
-import shutil
-import time
 import datetime
 
 import luigi

luigi/__init__.py

@@ -48,3 +48,14 @@
 from luigi.event import Event
 
 from .tools import range  # just makes the tool classes available from command line
+
+
+__all__ = [
+    'task', 'Task', 'Config', 'ExternalTask', 'WrapperTask', 'namespace',
+    'target', 'Target', 'File', 'LocalTarget', 'rpc', 'RemoteScheduler',
+    'RPCError', 'parameter', 'Parameter', 'DateParameter', 'MonthParameter',
+    'YearParameter', 'DateHourParameter', 'DateMinuteParameter', 'range',
+    'DateIntervalParameter', 'TimeDeltaParameter', 'IntParameter',
+    'FloatParameter', 'BooleanParameter', 'BoolParameter', 'TaskParameter',
+    'configuration', 'interface', 'file', 'run', 'build', 'event', 'Event'
+]

luigi/contrib/hadoop.py

@@ -304,7 +304,7 @@ def track_process(arglist, tracking_url_callback, env=None):
                 hadoop_context.job_id = job_id
 
         # Read the rest + stdout
-        err = ''.join(err_lines + [err_line for err_line in proc.stderr])
+        err = ''.join(err_lines + [an_err_line for an_err_line in proc.stderr])
         temp_stdout.seek(0)
         out = ''.join(temp_stdout.readlines())
 

luigi/contrib/hive.py

@@ -220,7 +220,6 @@ class HiveThriftContext(object):
 
     def __enter__(self):
         try:
-            from thrift import Thrift
             from thrift.transport import TSocket
             from thrift.transport import TTransport
             from thrift.protocol import TBinaryProtocol

luigi/contrib/pig.py

@@ -148,7 +148,7 @@ def track_and_progress(self, cmd):
                         logger.info(t)
 
         # Read the rest + stdout
-        err = ''.join(err_lines + [err_line.decode('utf8') for err_line in proc.stderr])
+        err = ''.join(err_lines + [an_err_line.decode('utf8') for an_err_line in proc.stderr])
         if proc.returncode == 0:
             logger.info("Job completed successfully!")
         else:

luigi/contrib/webhdfs.py

@@ -23,7 +23,6 @@
 
 import logging
 import os
-import sys
 
 from luigi import six
 

luigi/execution_summary.py

@@ -22,7 +22,6 @@
 """
 
 import textwrap
-import datetime
 
 
 def _partition_tasks(worker):
@@ -309,7 +308,6 @@ def _summary_format(set_tasks, worker):
     group_tasks = {}
     for status, task_dict in set_tasks.items():
         group_tasks[status] = _group_tasks_by_name_and_status(task_dict)
-    str_tasks = {}
     comments = _get_comments(group_tasks)
     num_all_tasks = len(set_tasks["already_done"]) + len(set_tasks["completed"]) + len(set_tasks["failed"]) + len(set_tasks["still_pending_ext"]) + len(set_tasks["still_pending_not_ext"])
     str_output = ''

luigi/hadoop.py

@@ -21,6 +21,6 @@
 
 import warnings
 
-from luigi.contrib.hadoop import *
+from luigi.contrib.hadoop import *  # NOQA
 warnings.warn("luigi.hadoop module has been moved to luigi.contrib.hadoop",
               DeprecationWarning)

luigi/hadoop_jar.py

@@ -21,7 +21,7 @@
 
 import warnings
 
-from luigi.contrib.hadoop_jar import *
+from luigi.contrib.hadoop_jar import *  # NOQA
 
 warnings.warn("luigi.hadoop_jar module has been moved to luigi.contrib.hadoop_jar",
               DeprecationWarning)

luigi/hdfs.py

@@ -21,7 +21,7 @@
 
 import warnings
 
-from luigi.contrib.hdfs import *
+from luigi.contrib.hdfs import *  # NOQA
 
 warnings.warn("luigi.hdfs module has been moved to luigi.contrib.hdfs",
               DeprecationWarning)

luigi/hive.py

@@ -21,7 +21,7 @@
 
 import warnings
 
-from luigi.contrib.hive import *
+from luigi.contrib.hive import *  # NOQA
 
 warnings.warn("luigi.hive module has been moved to luigi.contrib.hive",
               DeprecationWarning)

luigi/interface.py

@@ -238,8 +238,6 @@ def parse_task(self, cmdline_args=None):
 
         _add_global_parameters(parser)
 
-        task_names = Register.task_names()
-
         # Parse global arguments and pull out the task name.
         # We used to do this using subparsers+command, but some issues with
         # argparse across different versions of Python (2.7.9) made it hard.

luigi/rpc.py

@@ -25,7 +25,7 @@
 import socket
 import time
 
-from luigi.six.moves.urllib.parse import urljoin, urlencode, ParseResult
+from luigi.six.moves.urllib.parse import urljoin, urlencode
 from luigi.six.moves.urllib.request import urlopen
 from luigi.six.moves.urllib.error import URLError
 

luigi/scalding.py

@@ -20,6 +20,6 @@
 
 import warnings
 
-from luigi.contrib.scalding import *
+from luigi.contrib.scalding import *  # NOQA
 
 warnings.warn("luigi.scalding has now moved to luigi.contrib.scalding", DeprecationWarning, stacklevel=3)

luigi/scheduler.py

@@ -825,7 +825,7 @@ def _upstream_status(self, task_id, upstream_status_table):
                     elif upstream_status_table[dep_id] == '' and dep.deps:
                         # This is the postorder update step when we set the
                         # status based on the previously calculated child elements
-                        upstream_status = [upstream_status_table.get(task_id, '') for task_id in dep.deps]
+                        upstream_status = [upstream_status_table.get(a_task_id, '') for a_task_id in dep.deps]
                         upstream_status.append('')  # to handle empty list
                         status = max(upstream_status, key=UPSTREAM_SEVERITY_KEY)
                         upstream_status_table[dep_id] = status

luigi/server.py

@@ -53,7 +53,6 @@
 import tornado.netutil
 import tornado.web
 
-from luigi import configuration
 from luigi.scheduler import CentralPlannerScheduler
 
 

luigi/task.py

@@ -31,7 +31,7 @@
 from luigi import six
 
 from luigi import parameter
-from luigi.task_register import Register, TaskClassException
+from luigi.task_register import Register
 
 Parameter = parameter.Parameter
 logger = logging.getLogger('luigi-interface')

luigi/util.py

@@ -23,7 +23,7 @@
 
 from luigi import task
 from luigi import parameter
-from luigi.deprecate_kwarg import deprecate_kwarg  # removing this breaks code
+from luigi.deprecate_kwarg import deprecate_kwarg  # NOQA: removing this breaks code
 
 if six.PY3:
     xrange = range

luigi/webhdfs.py

@@ -20,7 +20,7 @@
 
 import warnings
 
-from luigi.contrib.webhdfs import *
+from luigi.contrib.webhdfs import *  # NOQA
 
 warnings.warn("luigi.webhdfs module has been moved to luigi.contrib.webhdfs",
               DeprecationWarning)

luigi/worker.py

@@ -49,7 +49,8 @@
 from luigi.task_register import load_task
 from luigi.scheduler import DISABLED, DONE, FAILED, PENDING, RUNNING, SUSPENDED, CentralPlannerScheduler
 from luigi.target import Target
-from luigi.task import Task, flatten, getpaths, TaskClassException, Config
+from luigi.task import Task, flatten, getpaths, Config
+from luigi.task_register import TaskClassException
 from luigi.parameter import FloatParameter, IntParameter, BoolParameter
 
 try:

setup.py

@@ -13,7 +13,6 @@
 # the License.
 
 import os
-import sys
 
 try:
     from setuptools import setup
@@ -22,7 +21,7 @@
 
 
 def get_static_files(path):
-    return [os.path.join(dirpath.replace("luigi/", ""), ext) 
+    return [os.path.join(dirpath.replace("luigi/", ""), ext)
             for (dirpath, dirnames, filenames) in os.walk(path)
             for ext in ["*.html", "*.js", "*.css", "*.png"]]
 

test/cmdline_test.py

@@ -49,7 +49,7 @@ class AmbiguousClass(luigi.Task):
     pass
 
 
-class AmbiguousClass(luigi.Task):
+class AmbiguousClass(luigi.Task):  # NOQA
     pass
 
 
@@ -59,7 +59,7 @@ def run(self):
         self.x = 42
 
 
-class TaskWithSameName(luigi.Task):
+class TaskWithSameName(luigi.Task):  # NOQA
     # there should be no ambiguity
 
     def run(self):

test/contrib/hadoop_test.py

@@ -381,7 +381,3 @@ def test_create_packages_archive_package_subpackage_submodule(self, tar):
         package_subpackage_submodule = __import__("package.subpackage.submodule", None, None, 'dummy')
         luigi.contrib.hadoop.create_packages_archive([package_subpackage_submodule], '/dev/null')
         self._assert_package_subpackage(tar.return_value.add)
-
-
-if __name__ == '__main__':
-    HadoopJobTest.test_run_real()

test/contrib/sqla_test.py

@@ -305,7 +305,7 @@ def output(self):
 
             def run(self):
                 out = self.output().open("w")
-                for task in TASK_LIST:
+                for task in self.TASK_LIST:
                     out.write("dummy_" + task)
                 out.close()
 
@@ -337,7 +337,7 @@ def copy(self, conn, ins_rows, table_bound):
                 conn.execute(ins, ins_rows)
 
             def rows(self):
-                for task in TASK_LIST:
+                for task in self.TASK_LIST:
                     yield task.strip("\n").split("\t")
 
         # Running only task1, and task2 should fail

test/parameter_test.py

@@ -16,14 +16,13 @@
 #
 
 import datetime
-from helpers import unittest, LuigiTestCase
+from helpers import unittest, with_config, LuigiTestCase
 from datetime import timedelta
 
 import luigi
 import luigi.date_interval
 import luigi.interface
 import luigi.notifications
-from helpers import with_config, LuigiTestCase
 from luigi.mock import MockTarget, MockFileSystem
 from luigi.parameter import ParameterException
 from worker_test import email_patch

test/redshift_test.py

@@ -31,7 +31,7 @@
 
 if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3):
     # spulec/moto#308
-    mock_s3 = unittest.skip('moto mock doesn\'t work with python3.4')
+    mock_s3 = unittest.skip('moto mock doesn\'t work with python3.4')  # NOQA
 
 
 luigi.notifications.DEBUG = True

test/server_test.py

@@ -99,7 +99,7 @@ def __init__(self):
         self.unix_socket = os.path.join(self.temp_dir, 'luigid.sock')
 
     def run_server(self):
-        luigi.server.run(unix_socket=unix_socket)
+        luigi.server.run(unix_socket=self.unix_socket)
 
     def scheduler(self):
         url = ParseResult(

test/worker_test.py

@@ -24,12 +24,11 @@
 import tempfile
 import threading
 import time
-from helpers import unittest, skipOnTravis
+from helpers import unittest, with_config, skipOnTravis
 
 import luigi.notifications
 import luigi.worker
 import mock
-from helpers import with_config, skipOnTravis
 from luigi import ExternalTask, RemoteScheduler, Task
 from luigi.mock import MockTarget, MockFileSystem
 from luigi.scheduler import CentralPlannerScheduler

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{27,33,34}-{cdh,hdp,nonhdfs,gcloud,postgres,unixsockets}, pypy-scheduler, docs, pep8
+envlist = py{27,33,34}-{cdh,hdp,nonhdfs,gcloud,postgres,unixsockets}, pypy-scheduler, docs, flake8
 skipsdist = True
 
 [testenv]
@@ -51,9 +51,11 @@ commands =
   coverage combine
   coveralls
 
-[testenv:pep8]
-deps = pep8
-commands = pep8 --exclude=luigi/six.py --ignore E501,E402,E731 luigi test examples bin
+[testenv:flake8]
+deps = flake8
+commands = flake8 --max-line-length=384 --exclude=doc,test,luigi/six.py
+  flake8 --max-line-length=100 --ignore=E265 doc
+  flake8 --max-line-length=252 --ignore=F401,F841 test
 
 [testenv:autopep8]
 deps = autopep8