Use literal syntax instead of function calls to create data structure (…

…#9516)

It is slower to call e.g. dict() than using the empty literal, because the name dict must be looked up in the global scope in case it has been rebound. Same for the other two types like list() and tuple().

.pre-commit-config.yaml

@@ -145,6 +145,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
     hooks:
       - id: check-merge-conflict
       - id: debug-statements
+      - id: check-builtin-literals
       - id: detect-private-key
       - id: end-of-file-fixer
       - id: mixed-line-ending

airflow/executors/base_executor.py

@@ -226,10 +226,10 @@ def get_event_buffer(self, dag_ids=None) -> Dict[TaskInstanceKeyType, EventBuffe
         :param dag_ids: to dag_ids to return events for, if None returns all
         :return: a dict of events
         """
-        cleared_events: Dict[TaskInstanceKeyType, EventBufferValueType] = dict()
+        cleared_events: Dict[TaskInstanceKeyType, EventBufferValueType] = {}
         if dag_ids is None:
             cleared_events = self.event_buffer
-            self.event_buffer = dict()
+            self.event_buffer = {}
         else:
             for key in list(self.event_buffer.keys()):
                 dag_id, _, _, _ = key

airflow/jobs/backfill_job.py

@@ -104,7 +104,7 @@ def __init__(self,  # pylint: disable=too-many-arguments
                      total_runs=0,
                      ):
             self.to_run = to_run or OrderedDict()
-            self.running = running or dict()
+            self.running = running or {}
             self.skipped = skipped or set()
             self.succeeded = succeeded or set()
             self.failed = failed or set()

airflow/models/dag.py

@@ -257,7 +257,7 @@ def __init__(
         # set file location to caller source path
         back = sys._getframe().f_back
         self.fileloc = back.f_code.co_filename if back else ""
-        self.task_dict: Dict[str, BaseOperator] = dict()
+        self.task_dict: Dict[str, BaseOperator] = {}
 
         # set timezone from start_date
         if start_date and start_date.tzinfo:
@@ -1277,7 +1277,7 @@ def get_task(self, task_id, include_subdags=False):
         raise TaskNotFound("Task {task_id} not found".format(task_id=task_id))
 
     def pickle_info(self):
-        d = dict()
+        d = {}
         d['is_picklable'] = True
         try:
             dttm = timezone.utcnow()

airflow/models/dagcode.py

@@ -92,7 +92,7 @@ def bulk_sync_to_db(cls, filelocs: Iterable[str], session=None):
                 orm_dag_code.fileloc: orm_dag_code for orm_dag_code in existing_orm_dag_codes
             }
         else:
-            existing_orm_dag_codes_map = dict()
+            existing_orm_dag_codes_map = {}
 
         existing_orm_dag_codes_by_fileloc_hashes = {
             orm.fileloc_hash: orm for orm in existing_orm_dag_codes

airflow/providers/amazon/aws/hooks/base_aws.py

@@ -89,7 +89,7 @@ def _get_credentials(self, region_name):
         aws_secret_access_key = None
         aws_session_token = None
         endpoint_url = None
-        session_kwargs = dict()
+        session_kwargs = {}
 
         if self.aws_conn_id:  # pylint: disable=too-many-nested-blocks
             self.log.info("Airflow Connection: aws_conn_id=%s",
@@ -187,7 +187,7 @@ def _get_credentials(self, region_name):
                     )
                     sts_client = sts_session.client("sts", config=self.config)
 
-                    assume_role_kwargs = dict()
+                    assume_role_kwargs = {}
                     if "assume_role_kwargs" in extra_config:
                         assume_role_kwargs = extra_config["assume_role_kwargs"]
 

airflow/providers/amazon/aws/hooks/sagemaker.py

@@ -763,7 +763,7 @@ def list_training_jobs(
         :return: results of the list_training_jobs request
         """
 
-        config = dict()
+        config = {}
 
         if name_contains:
             if "NameContains" in kwargs:
@@ -806,7 +806,7 @@ def _list_request(self, partial_func, result_key: str, max_results: Optional[int
         next_token = None
 
         while True:
-            kwargs = dict()
+            kwargs = {}
             if next_token is not None:
                 kwargs["NextToken"] = next_token
 

airflow/providers/amazon/aws/operators/datasync.py

@@ -137,16 +137,16 @@ def __init__(
         self.allow_random_task_choice = allow_random_task_choice
         self.allow_random_location_choice = allow_random_location_choice
 
-        self.create_task_kwargs = create_task_kwargs if create_task_kwargs else dict()
-        self.create_source_location_kwargs = dict()
+        self.create_task_kwargs = create_task_kwargs if create_task_kwargs else {}
+        self.create_source_location_kwargs = {}
         if create_source_location_kwargs:
             self.create_source_location_kwargs = create_source_location_kwargs
-        self.create_destination_location_kwargs = dict()
+        self.create_destination_location_kwargs = {}
         if create_destination_location_kwargs:
             self.create_destination_location_kwargs = create_destination_location_kwargs
 
-        self.update_task_kwargs = update_task_kwargs if update_task_kwargs else dict()
-        self.task_execution_kwargs = task_execution_kwargs if task_execution_kwargs else dict()
+        self.update_task_kwargs = update_task_kwargs if update_task_kwargs else {}
+        self.task_execution_kwargs = task_execution_kwargs if task_execution_kwargs else {}
         self.delete_task_after_execution = delete_task_after_execution
 
         # Validations

airflow/providers/google/cloud/operators/bigquery.py

@@ -1042,7 +1042,7 @@ def __init__(
         self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
         self.delegate_to = delegate_to
 
-        self.src_fmt_configs = src_fmt_configs or dict()
+        self.src_fmt_configs = src_fmt_configs or {}
         self.labels = labels
         self.encryption_configuration = encryption_configuration
         self.location = location

airflow/providers/google/cloud/operators/bigtable.py

@@ -255,7 +255,7 @@ def __init__(self,
         self.instance_id = instance_id
         self.table_id = table_id
         self.initial_split_keys = initial_split_keys or []
-        self.column_families = column_families or dict()
+        self.column_families = column_families or {}
         self._validate_inputs()
         self.gcp_conn_id = gcp_conn_id
         super().__init__(*args, **kwargs)

airflow/providers/google/cloud/operators/dataproc.py

@@ -208,7 +208,7 @@ def __init__(self,
         self.custom_image = custom_image
         self.custom_image_project_id = custom_image_project_id
         self.image_version = image_version
-        self.properties = properties or dict()
+        self.properties = properties or {}
         self.optional_components = optional_components
         self.master_machine_type = master_machine_type
         self.master_disk_type = master_disk_type

airflow/providers/google/cloud/transfers/cassandra_to_gcs.py

@@ -307,7 +307,7 @@ def generate_schema_dict(cls, name: str, type_: Any) -> Dict[str, Any]:
         """
         Generates BQ schema.
         """
-        field_schema: Dict[str, Any] = dict()
+        field_schema: Dict[str, Any] = {}
         field_schema.update({'name': name})
         field_schema.update({'type_': cls.get_bq_type(type_)})
         field_schema.update({'mode': cls.get_bq_mode(type_)})

airflow/providers/google/cloud/utils/credentials_provider.py

@@ -59,7 +59,7 @@ def build_gcp_conn(
     conn = "google-cloud-platform://?{}"
     extras = "extra__google_cloud_platform"
 
-    query_params = dict()
+    query_params = {}
     if key_file_path:
         query_params["{}__key_path".format(extras)] = key_file_path
     if scopes:

airflow/utils/log/file_processor_handler.py

@@ -84,7 +84,7 @@ def close(self):
 
     def _render_filename(self, filename):
         filename = os.path.relpath(filename, self.dag_dir)
-        ctx = dict()
+        ctx = {}
         ctx['filename'] = filename
 
         if self.filename_jinja_template:

airflow/utils/log/logging_mixin.py

@@ -72,7 +72,7 @@ def __init__(self, logger, level):
         """
         self.logger = logger
         self.level = level
-        self._buffer = str()
+        self._buffer = ''
 
     @property
     def closed(self):   # noqa: D402
@@ -101,15 +101,15 @@ def write(self, message):
         else:
             self._buffer += message
             self._propagate_log(self._buffer.rstrip())
-            self._buffer = str()
+            self._buffer = ''
 
     def flush(self):
         """
         Ensure all logging output has been flushed
         """
         if len(self._buffer) > 0:
             self._propagate_log(self._buffer)
-            self._buffer = str()
+            self._buffer = ''
 
     def isatty(self):
         """

airflow/utils/operator_helpers.py

@@ -46,7 +46,7 @@ def context_to_airflow_vars(context, in_env_var_format=False):
     :type in_env_var_format: bool
     :return: task_instance context as dict.
     """
-    params = dict()
+    params = {}
     if in_env_var_format:
         name_format = 'env_var_format'
     else:

airflow/www/extensions/init_manifest_files.py

@@ -29,7 +29,7 @@ def configure_manifest_files(app):
     :param app:
     :return:
     """
-    manifest = dict()
+    manifest = {}
 
     def parse_manifest_json():
         # noinspection PyBroadException

backport_packages/setup_backport_packages.py

@@ -604,7 +604,7 @@ def get_package_class_summary(full_package_name: str, imported_classes: List[str
     from airflow.hooks.base_hook import BaseHook
     from airflow.models.baseoperator import BaseOperator
 
-    all_verified_entities: Dict[EntityType, VerifiedEntities] = dict()
+    all_verified_entities: Dict[EntityType, VerifiedEntities] = {}
     all_verified_entities[EntityType.Operators] = find_all_entities(
         imported_classes=imported_classes,
         base_package=full_package_name,
@@ -657,7 +657,7 @@ def get_package_class_summary(full_package_name: str, imported_classes: List[str
     for entity in EntityType:
         print_wrong_naming(entity, all_verified_entities[entity].wrong_entities)
 
-    entities_summary: Dict[EntityType, EntityTypeSummary] = dict() # noqa
+    entities_summary: Dict[EntityType, EntityTypeSummary] = {} # noqa
 
     for entity_type in EntityType:
         entities_summary[entity_type] = get_details_about_classes(

dev/send_email.py

@@ -127,7 +127,7 @@ def __init__(
         self.password = password
         self.version = version
         self.version_rc = version_rc
-        self.template_arguments = dict()
+        self.template_arguments = {}
 
     def __repr__(self):
         return f"Apache Credentials: {self.email}/{self.username}/{self.version}/{self.version_rc}"

tests/models/test_dag.py

@@ -317,7 +317,7 @@ def test_dag_topological_sort_dag_without_tasks(self):
             start_date=DEFAULT_DATE,
             default_args={'owner': 'owner1'})
 
-        self.assertEqual(tuple(), dag.topological_sort())
+        self.assertEqual((), dag.topological_sort())
 
     def test_dag_naive_start_date_string(self):
         DAG('DAG', default_args={'start_date': '2019-06-01'})

tests/test_stats.py

@@ -74,7 +74,7 @@ def test_increment_counter_with_valid_name(self):
         self.statsd_client.incr.assert_called_once_with('test_stats_run', 1, 1)
 
     def test_stat_name_must_be_a_string(self):
-        self.stats.incr(list())
+        self.stats.incr([])
         self.statsd_client.assert_not_called()
 
     def test_stat_name_must_not_exceed_max_length(self):
@@ -152,7 +152,7 @@ def test_increment_counter_with_valid_name_with_dogstatsd(self):
         )
 
     def test_stat_name_must_be_a_string_with_dogstatsd(self):
-        self.dogstatsd.incr(list())
+        self.dogstatsd.incr([])
         self.dogstatsd_client.assert_not_called()
 
     def test_stat_name_must_not_exceed_max_length_with_dogstatsd(self):

tests/utils/log/elasticmock/fake_elasticsearch.py

@@ -81,7 +81,7 @@ def info(self, params=None):
                   'routing', 'timeout', 'timestamp', 'ttl', 'version', 'version_type')
     def index(self, index, doc_type, body, id=None, params=None):
         if index not in self.__documents_dict:
-            self.__documents_dict[index] = list()
+            self.__documents_dict[index] = []
 
         if id is None:
             id = get_random_id()