[Dataproc] Pyspark Job with Airrflow (Composer): Get Data From MySQL

UBUNTU CLIENT CONFIGURATION export TEMPLATE_ID=workflow-mytest export WORK_CLUSTER_NAME=cluster-mytest export REGION=asia-northeast3 export BUCKET_NAME=jay-pyspark-mytest #airflow task dag에서도 필요 export PROJECT_ID= #airflow task dag에서도 필요 export PYTHON_FILE=pyspark-job.py export STEP_ID=first_step #Some name like "Get Data" PYTHON CODE $vi pywork.py import pymysql import sys import pandas as pd f..

2021.11.01