Python postgres to s3 script

From Federal Burro of Information
Jump to navigationJump to search
import sys, os
import pgpasslib
import psycopg2

passw = pgpasslib.getpass('servername',
  5439,
  'db_name',
  'user')
print('connecting')
conn = psycopg2.connect(
    host='servername',
    user='user',
      dbname='db_name',
      password=passw,
      port=5439,
      connect_timeout=500)
process_date = sys.argv[1]
formatted_date =  process_date.replace('-','_')
for file in os.listdir("."):
  if file.endswith(".sql"):
    with open(file,"a") as sql_file:
      print('-- ' + file)
      filename = file.split('.')[0].replace('_ddl','')
      partition_str = """
      alter table spectrum.table_%(n)s               
      add partition(logdate='%(d)s')                
      location 's3://bucketname/PuD5v0L6Q5adq4ya1w2PKw/ec53758d-1e9c-400e-8468-ce4cd25d9c48/S3_CSV/%(u)s/%(f)s';
      """ % {"d": process_date, "f": formatted_date, "n": filename, "u": filename.upper()}
      print(partition_str)   
      sql_file.write(partition_str)