[AIRFLOW-6139] Consistent spaces in pylint enable/disable (#6701)

.pre-commit-config.yaml

@@ -204,6 +204,12 @@ repos:
         entry: "pydevd.*settrace\\("
         pass_filenames: true
         files: \.py$
+      - id: consistent-pylint
+        language: pygrep
+        name: Check for inconsitent pylint disable/enable without space
+        entry: "pylint:disable|pylint:enable"
+        pass_filenames: true
+        files: \.py$
       - id: base-operator
         language: pygrep
         name: Make sure BaseOperator is imported from airflow.models.baseoperator in core

CONTRIBUTING.rst

@@ -318,7 +318,7 @@ These are guidelines for fixing errors reported by pylint:
 
 
 -   For multiple lines/block of code, to disable an error, you can surround the
-    block with ``pylint:disable/pylint:enable`` comment lines. For example:
+    block with ``pylint: disable/pylint: enable`` comment lines. For example:
 
 .. code-block:: python
 
@@ -428,6 +428,8 @@ image built locally):
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``check-xml``                         Checks XML files with xmllint.
 ----------------------------------- ---------------------------------------------------------------- ------------
+``consistent-pylint``                 Consistent usage of pylint enable/disable with space.
+----------------------------------- ---------------------------------------------------------------- ------------
 ``debug-statements``                  Detects accidenatally committed debug statements.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``detect-private-key``                Detects if private key is added to the repository.

airflow/__init__.py

@@ -28,7 +28,7 @@
 """
 
 # flake8: noqa: F401
-# pylint:disable=wrong-import-position
+# pylint: disable=wrong-import-position
 from typing import Callable, Optional
 
 # noinspection PyUnresolvedReferences
@@ -54,13 +54,13 @@
 
 
 class AirflowMacroPlugin:
-    # pylint:disable=missing-docstring
+    # pylint: disable=missing-docstring
     def __init__(self, namespace):
         self.namespace = namespace
 
 
-operators._integrate_plugins()  # pylint:disable=protected-access
-sensors._integrate_plugins()  # pylint:disable=protected-access
-hooks._integrate_plugins()  # pylint:disable=protected-access
-executors._integrate_plugins()  # pylint:disable=protected-access
-macros._integrate_plugins()  # pylint:disable=protected-access
+operators._integrate_plugins()  # pylint: disable=protected-access
+sensors._integrate_plugins()  # pylint: disable=protected-access
+hooks._integrate_plugins()  # pylint: disable=protected-access
+executors._integrate_plugins()  # pylint: disable=protected-access
+macros._integrate_plugins()  # pylint: disable=protected-access

airflow/contrib/utils/log/task_handler_with_custom_formatter.py

@@ -50,7 +50,7 @@ def set_context(self, ti):
         if prefix:
             _, self.prefix_jinja_template = parse_template_string(prefix)
             rendered_prefix = self._render_prefix(ti)
-        formatter = logging.Formatter(rendered_prefix + ":" + self.formatter._fmt)  # pylint:disable=W0212
+        formatter = logging.Formatter(rendered_prefix + ":" + self.formatter._fmt)  # pylint: disable=W0212
         self.setFormatter(formatter)
         self.setLevel(self.level)
 

airflow/executors/__init__.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# pylint:disable=missing-docstring
+# pylint: disable=missing-docstring
 
 import sys
 from typing import Optional
@@ -37,12 +37,12 @@ def _integrate_plugins():
     from airflow.plugins_manager import executors_modules
     for executors_module in executors_modules:
         sys.modules[executors_module.__name__] = executors_module
-        globals()[executors_module._name] = executors_module  # pylint:disable=protected-access
+        globals()[executors_module._name] = executors_module  # pylint: disable=protected-access
 
 
 def get_default_executor():
     """Creates a new instance of the configured executor if none exists and returns it"""
-    global DEFAULT_EXECUTOR  # pylint:disable=global-statement
+    global DEFAULT_EXECUTOR  # pylint: disable=global-statement
 
     if DEFAULT_EXECUTOR is not None:
         return DEFAULT_EXECUTOR

airflow/gcp/example_dags/example_mlengine.py

@@ -201,7 +201,7 @@ def validate_err_and_count(summary: Dict) -> Dict:
         return summary
 
     evaluate_prediction, evaluate_summary, evaluate_validation = mlengine_operator_utils.create_evaluate_ops(
-        task_prefix="evalueate-ops",  # pylint:disable=too-many-arguments
+        task_prefix="evalueate-ops",  # pylint: disable=too-many-arguments
         data_format="TEXT",
         input_paths=[PREDICTION_INPUT],
         prediction_path=PREDICTION_OUTPUT,

airflow/gcp/hooks/automl.py

@@ -335,7 +335,7 @@ def import_data(
 
     @GoogleCloudBaseHook.catch_http_exception
     @GoogleCloudBaseHook.fallback_to_default_project_id
-    def list_column_specs(  # pylint:disable=too-many-arguments
+    def list_column_specs(  # pylint: disable=too-many-arguments
         self,
         dataset_id: str,
         table_spec_id: str,

airflow/gcp/hooks/datastore.py

@@ -88,7 +88,7 @@ def allocate_ids(self, partial_keys: List, project_id: Optional[str] = None) ->
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .allocateIds(projectId=project_id, body={'keys': partial_keys})
                 .execute(num_retries=self.num_retries))
@@ -110,7 +110,7 @@ def begin_transaction(self, project_id: Optional[str] = None) -> str:
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .beginTransaction(projectId=project_id, body={})
                 .execute(num_retries=self.num_retries))
@@ -134,7 +134,7 @@ def commit(self, body: Dict, project_id: Optional[str] = None) -> Dict:
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .commit(projectId=project_id, body=body)
                 .execute(num_retries=self.num_retries))
@@ -172,7 +172,7 @@ def lookup(self,
             body['readConsistency'] = read_consistency
         if transaction:
             body['transaction'] = transaction
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .lookup(projectId=project_id, body=body)
                 .execute(num_retries=self.num_retries))
@@ -194,7 +194,7 @@ def rollback(self, transaction: str, project_id: Optional[str] = None) -> Any:
         """
         conn = self.get_conn()  # type: Any
 
-        conn.projects().rollback(  # pylint:disable=no-member
+        conn.projects().rollback(  # pylint: disable=no-member
             projectId=project_id, body={'transaction': transaction}
         ).execute(num_retries=self.num_retries)
 
@@ -215,7 +215,7 @@ def run_query(self, body: Dict, project_id: Optional[str] = None) -> Dict:
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .runQuery(projectId=project_id, body=body)
                 .execute(num_retries=self.num_retries))
@@ -236,7 +236,7 @@ def get_operation(self, name: str) -> Dict:
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .operations()
                 .get(name=name)
@@ -258,7 +258,7 @@ def delete_operation(self, name: str) -> Dict:
         """
         conn = self.get_conn()  # type: Any
 
-        resp = (conn  # pylint:disable=no-member
+        resp = (conn  # pylint: disable=no-member
                 .projects()
                 .operations()
                 .delete(name=name)
@@ -329,7 +329,7 @@ def export_to_storage_bucket(self,
             'entityFilter': entity_filter,
             'labels': labels,
         }  # type: Dict
-        resp = (admin_conn  # pylint:disable=no-member
+        resp = (admin_conn  # pylint: disable=no-member
                 .projects()
                 .export(projectId=project_id, body=body)
                 .execute(num_retries=self.num_retries))
@@ -380,7 +380,7 @@ def import_from_storage_bucket(self,
             'entityFilter': entity_filter,
             'labels': labels,
         }  # type: Dict
-        resp = (admin_conn  # pylint:disable=no-member
+        resp = (admin_conn  # pylint: disable=no-member
                 .projects()
                 .import_(projectId=project_id, body=body)
                 .execute(num_retries=self.num_retries))

airflow/gcp/hooks/functions.py

@@ -209,7 +209,7 @@ def call_function(
             location=location,
             function_id=function_id
         )
-        response = self.get_conn().projects().locations().functions().call(  # pylint:disable=no-member
+        response = self.get_conn().projects().locations().functions().call(  # pylint: disable=no-member
             name=name,
             body=input_data
         ).execute(num_retries=self.num_retries)

airflow/gcp/operators/automl.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# pylint:disable=too-many-lines
+# pylint: disable=too-many-lines
 """
 This module contains Google AutoML operators.
 """
@@ -232,7 +232,7 @@ class AutoMLBatchPredictOperator(BaseOperator):
     )
 
     @apply_defaults
-    def __init__(  # pylint:disable=too-many-arguments
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         model_id: str,
         input_config: dict,
@@ -479,7 +479,7 @@ class AutoMLTablesListColumnSpecsOperator(BaseOperator):
     )
 
     @apply_defaults
-    def __init__(  # pylint:disable=too-many-arguments
+    def __init__(  # pylint: disable=too-many-arguments
         self,
         dataset_id: str,
         table_spec_id: str,

airflow/gcp/operators/bigquery.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# pylint:disable=too-many-lines
+# pylint: disable=too-many-lines
 """
 This module contains Google BigQuery operators.
 """

airflow/gcp/operators/datastore.py

@@ -63,7 +63,7 @@ class DatastoreExportOperator(BaseOperator):
     template_fields = ['bucket', 'namespace', 'entity_filter', 'labels']
 
     @apply_defaults
-    def __init__(self,  # pylint:disable=too-many-arguments
+    def __init__(self,  # pylint: disable=too-many-arguments
                  bucket: str,
                  namespace: Optional[str] = None,
                  datastore_conn_id: str = 'google_cloud_default',

airflow/gcp/operators/mlengine.py

@@ -162,7 +162,7 @@ class MLEngineBatchPredictionOperator(BaseOperator):
     ]
 
     @apply_defaults
-    def __init__(self,  # pylint:disable=too-many-arguments
+    def __init__(self,  # pylint: disable=too-many-arguments
                  job_id: str,
                  region: str,
                  data_format: str,
@@ -916,7 +916,7 @@ class MLEngineTrainingOperator(BaseOperator):
     ]
 
     @apply_defaults
-    def __init__(self,  # pylint:disable=too-many-arguments
+    def __init__(self,  # pylint: disable=too-many-arguments
                  job_id: str,
                  package_uris: List[str],
                  training_python_module: str,

airflow/gcp/sensors/gcs.py

@@ -53,7 +53,7 @@ class GoogleCloudStorageObjectSensor(BaseSensorOperator):
     @apply_defaults
     def __init__(self,
                  bucket: str,
-                 object: str,  # pylint:disable=redefined-builtin
+                 object: str,  # pylint: disable=redefined-builtin
                  google_cloud_conn_id: str = 'google_cloud_default',
                  delegate_to: Optional[str] = None,
                  *args, **kwargs) -> None:
@@ -108,7 +108,7 @@ class GoogleCloudStorageObjectUpdatedSensor(BaseSensorOperator):
     @apply_defaults
     def __init__(self,
                  bucket: str,
-                 object: str,  # pylint:disable=redefined-builtin
+                 object: str,  # pylint: disable=redefined-builtin
                  ts_func: Callable = ts_function,
                  google_cloud_conn_id: str = 'google_cloud_default',
                  delegate_to: Optional[str] = None,

airflow/gcp/utils/mlengine_operator_utils.py

@@ -35,7 +35,7 @@
 from airflow.operators.python_operator import PythonOperator
 
 
-def create_evaluate_ops(task_prefix,  # pylint:disable=too-many-arguments
+def create_evaluate_ops(task_prefix,  # pylint: disable=too-many-arguments
                         data_format,
                         input_paths,
                         prediction_path,

airflow/hooks/__init__.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# pylint:disable=missing-docstring
+# pylint: disable=missing-docstring
 
 import sys
 
@@ -30,4 +30,4 @@ def _integrate_plugins():
     from airflow.plugins_manager import hooks_modules
     for hooks_module in hooks_modules:
         sys.modules[hooks_module.__name__] = hooks_module
-        globals()[hooks_module._name] = hooks_module  # pylint:disable=protected-access
+        globals()[hooks_module._name] = hooks_module  # pylint: disable=protected-access

airflow/jobs/__init__.py

@@ -18,7 +18,7 @@
 # under the License.
 #
 
-# pylint:disable=missing-docstring
+# pylint: disable=missing-docstring
 
 from airflow.jobs.backfill_job import BackfillJob  # noqa: F401
 from airflow.jobs.base_job import BaseJob  # noqa: F401

airflow/lineage/__init__.py

@@ -123,7 +123,7 @@ def prepare_lineage(func):
     * "list of datasets" -> manually defined list of data
 
     """
-    # pylint:disable=protected-access
+    # pylint: disable=protected-access
     @wraps(func)
     def wrapper(self, context, *args, **kwargs):
         self.log.debug("Preparing lineage inlets and outlets")

airflow/macros/__init__.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# pylint:disable=missing-docstring
+# pylint: disable=missing-docstring
 
 import time  # noqa
 import uuid  # noqa
@@ -93,4 +93,4 @@ def _integrate_plugins():
     from airflow.plugins_manager import macros_modules
     for macros_module in macros_modules:
         sys.modules[macros_module.__name__] = macros_module
-        globals()[macros_module._name] = macros_module  # pylint:disable=protected-access
+        globals()[macros_module._name] = macros_module  # pylint: disable=protected-access

airflow/models/__init__.py

@@ -24,7 +24,7 @@
 from airflow.models.dagbag import DagBag  # noqa: F401
 from airflow.models.dagpickle import DagPickle  # noqa: F401
 from airflow.models.dagrun import DagRun  # noqa: F401
-from airflow.models.errors import ImportError  # noqa: F401, pylint:disable=redefined-builtin
+from airflow.models.errors import ImportError  # noqa: F401, pylint: disable=redefined-builtin
 from airflow.models.kubernetes import KubeResourceVersion, KubeWorkerIdentifier  # noqa: F401
 from airflow.models.log import Log  # noqa: F401
 from airflow.models.pool import Pool  # noqa: F401

airflow/operators/__init__.py

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# pylint:disable=missing-docstring
+# pylint: disable=missing-docstring
 
 
 def _integrate_plugins():
@@ -26,4 +26,4 @@ def _integrate_plugins():
     from airflow.plugins_manager import operators_modules
     for operators_module in operators_modules:
         sys.modules[operators_module.__name__] = operators_module
-        globals()[operators_module._name] = operators_module  # pylint:disable=protected-access
+        globals()[operators_module._name] = operators_module  # pylint: disable=protected-access

airflow/operators/bigquery_to_bigquery.py

@@ -76,7 +76,7 @@ class BigQueryToBigQueryOperator(BaseOperator):
     ui_color = '#e6f0e4'
 
     @apply_defaults
-    def __init__(self,  # pylint:disable=too-many-arguments
+    def __init__(self,  # pylint: disable=too-many-arguments
                  source_project_dataset_tables: Union[List[str], str],
                  destination_project_dataset_table: str,
                  write_disposition: str = 'WRITE_EMPTY',

airflow/operators/bigquery_to_mysql.py

@@ -79,7 +79,7 @@ class BigQueryToMySqlOperator(BaseOperator):
     template_fields = ('dataset_id', 'table_id', 'mysql_table')
 
     @apply_defaults
-    def __init__(self,  # pylint:disable=too-many-arguments
+    def __init__(self,  # pylint: disable=too-many-arguments
                  dataset_table: str,
                  mysql_table: str,
                  selected_fields: Optional[str] = None,

airflow/providers/apache/cassandra/hooks/cassandra.py

@@ -207,5 +207,5 @@ def record_exists(self, table: str, keys: Dict[str, str]) -> bool:
         try:
             result = self.get_conn().execute(query, keys)
             return result.one() is not None
-        except Exception:  # pylint:disable=broad-except
+        except Exception:  # pylint: disable=broad-except
             return False