Python Script로 Sqoop import 하는 방법(MySQL --> HDFS)

https://medium.com/@sandeepsinh/sqoop-jobs-development-using-python-82e4b0139457

Sqoop job development using Python

기존의 경로에 파일을 있다면 에러가 나기에 파일을 삭제하고 다시 만들어야 한다.

--delete-target-dir --target-dir 'user/hadoop/input

import subprocess
import logging
 
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
   # Function to run Hadoop command
def run_unix_cmd(args_list):
 """
    run linux commands
    """
    print('Running system command:{0}'.format('     '.join(args_list)))
    proc = subprocess.Popen(args_list, stdout=subprocess.PIPE,         stderr=subprocess.PIPE, universal_newlines=True)
    s_output, s_err = proc.communicate()
    s_return = proc.returncode
    return s_return, s_output, s_err
 
  # Create Sqoop Job
def sqoop_job():
   """
   Create Sqoop job
   """
   cmd = ['sqoop', 'import', '--connect', 'jdbc:mysql://192.168.0.35:3309/member?serverTimezone=Asia/Seoul', '--username', 'hadoop','--password', 'markate123', '--tab    le', 'student', '-m', '1', '--delete-target-dir','--target-dir', '/user/hadoop/input']
   print(cmd)
   (ret, out, err) = run_unix_cmd(cmd)
   print(ret, out, err)
   if ret == 0:
      logging.info('Success.')
   else:
      logging.info('Error.')
 
if __name__ == '__main__':
    sqoop_job()