Python postgres to s3 script
From Federal Burro of Information
Jump to navigationJump to search
import sys, os import pgpasslib import psycopg2 passw = pgpasslib.getpass('servername', 5439, 'db_name', 'user') print('connecting') conn = psycopg2.connect( host='servername', user='user', dbname='db_name', password=passw, port=5439, connect_timeout=500) process_date = sys.argv[1] formatted_date = process_date.replace('-','_') for file in os.listdir("."): if file.endswith(".sql"): with open(file,"a") as sql_file: print('-- ' + file) filename = file.split('.')[0].replace('_ddl','') partition_str = """ alter table spectrum.table_%(n)s add partition(logdate='%(d)s') location 's3://bucketname/PuD5v0L6Q5adq4ya1w2PKw/ec53758d-1e9c-400e-8468-ce4cd25d9c48/S3_CSV/%(u)s/%(f)s'; """ % {"d": process_date, "f": formatted_date, "n": filename, "u": filename.upper()} print(partition_str) sql_file.write(partition_str)