Acabei usando o blog de Andre Dunstan e esta resposta SO que diz para formatar o json de uma maneira específica para usar o comando copy.
Como minha estrutura está bem definida para os arquivos que estou analisando, acabei com o seguinte script.
def file_len(fname):
# to find the number of lines in the file.
# Has been pretty efficient even for millions of records
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
INPUTFILE = '/path/to/input.json'
OUTPUTFILE = '/path/to/output.json.csv'
LEN = file_len(INPUTFILE)
with open(OUTPUTFILE, 'w') as fo:
with open(INPUTFILE, 'r') as fi:
for i, l in enumerate(fi):
# I skip the first line
if i == 0: continue
# To remove the ']}}' from the end
elif i+1 == LEN: _ = fo.write(l[:-3])
# To remove the ',' from the end
# and add \n since write does not add newline on its own
else: _ = fo.write(l[:-2]+'\n')
# load statement
import sqlalchemy
POSTGRESQL = f'postgresql+psycopg2://{USERNAME}:{PASSWORD}@{HOSTNAME}/{DB}'
engine = sqlalchemy.create_engine(POSTGRESQL, echo=True)
con = engine.connect()
trans = con.begin()
LOAD_SQL = f"COPY tablename from '{OUTPUTFILE}' with csv delimiter E'\x01' quote E'\x02' null as '';"
try:
con.execute(LOAD_SQL)
trans.commit()
except Exception as e:
trans.rollback()
finally:
con.close()
