[AIRFLOW-5691] Rewrite Dataproc operators to use python library (#6371)

apache · Nov 13, 2019 · d633d3a · d633d3a
1 parent 2897624
commit d633d3a
Show file tree

Hide file tree

Showing 15 changed files with 3,175 additions and 2,792 deletions.
diff --git a/airflow/contrib/hooks/gcp_dataproc_hook.py b/airflow/contrib/hooks/gcp_dataproc_hook.py
@@ -16,27 +16,28 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""This module is deprecated. Please use `airflow.gcp.hooks.dataproc`."""
+"""This module is deprecated. Please use `airflow.providers.google.cloud.hooks.dataproc`."""
 
 import warnings
 
 # pylint: disable=unused-import
-from airflow.gcp.hooks.dataproc import DataprocHook, DataprocJobStatus  # noqa
+from airflow.providers.google.cloud.hooks.dataproc import DataprocHook  # noqa
 
 warnings.warn(
-    "This module is deprecated. Please use `airflow.gcp.hooks.dataproc`.",
+    "This module is deprecated. Please use `airflow.providers.google.cloud.hooks.dataproc`.",
     DeprecationWarning, stacklevel=2
 )
 
 
 class DataProcHook(DataprocHook):
     """
-    This class is deprecated. Please use `airflow.gcp.hooks.dataproc.DataprocHook`.
+    This class is deprecated. Please use `airflow.providers.google.cloud.hooks.dataproc.DataprocHook`.
     """
 
     def __init__(self, *args, **kwargs):
         warnings.warn(
-            "This class is deprecated. Please use `airflow.gcp.hooks.dataproc.DataprocHook`.",
+            "This class is deprecated. Please use `airflow.providers.google."
+            "cloud.hooks.dataproc.DataprocHook`.",
             DeprecationWarning, stacklevel=2
         )
 

diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
@@ -16,19 +16,21 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""This module is deprecated. Please use `airflow.gcp.operators.dataproc`."""
+"""This module is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc`."""
 
 import warnings
 
 # pylint: disable=unused-import
-from airflow.gcp.operators.dataproc import (  # noqa
+from airflow.providers.google.cloud.operators.dataproc import (  # noqa
     DataprocClusterCreateOperator, DataprocClusterDeleteOperator, DataprocClusterScaleOperator,
-    DataProcHadoopOperator, DataProcHiveOperator, DataProcJobBaseOperator, DataprocOperationBaseOperator,
+    DataProcHadoopOperator, DataProcHiveOperator, DataProcJobBaseOperator, DataProcJobBuilder,
     DataProcPigOperator, DataProcPySparkOperator, DataProcSparkOperator, DataProcSparkSqlOperator,
+    DataprocSubmitJobOperator, DataprocUpdateClusterOperator,
     DataprocWorkflowTemplateInstantiateInlineOperator, DataprocWorkflowTemplateInstantiateOperator,
 )
 
 warnings.warn(
-    "This module is deprecated. Please use `airflow.gcp.operators.dataproc`.",
-    DeprecationWarning, stacklevel=2
+    "This module is deprecated. Please use `airflow.providers.google.cloud.operators.dataproc`.",
+    DeprecationWarning,
+    stacklevel=2,
 )
diff --git a/airflow/gcp/example_dags/example_dataproc.py b/airflow/gcp/example_dags/example_dataproc.py