Skip to content

Commit

Permalink
Make the port script drop NUL values in all tables
Browse files Browse the repository at this point in the history
Postgres doesn't support NULs in strings so it makes the script
throw an exception and stop if any values contain \0. Drop them
with appropriate warning.
  • Loading branch information
dbkr committed Oct 31, 2017
1 parent c31a7c3 commit 9d419f4
Showing 1 changed file with 19 additions and 7 deletions.
26 changes: 19 additions & 7 deletions scripts/synapse_port_db
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ class Porter(object):
backward_chunk = min(row[0] for row in brows) - 1

rows = frows + brows
self._convert_rows(table, headers, rows)
rows = self._convert_rows(table, headers, rows)

def insert(txn):
self.postgres_store.insert_many_txn(
Expand Down Expand Up @@ -556,17 +556,29 @@ class Porter(object):
i for i, h in enumerate(headers) if h in bool_col_names
]

class BadValueException(Exception):
pass

def conv(j, col):
if j in bool_cols:
return bool(col)
elif isinstance(col, basestring) and "\0" in col:
logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
raise BadValueException();
return col

outrows = []
for i, row in enumerate(rows):
rows[i] = tuple(
conv(j, col)
for j, col in enumerate(row)
if j > 0
)
try:
outrows.append(tuple(
conv(j, col)
for j, col in enumerate(row)
if j > 0
))
except BadValueException:
pass

return outrows

@defer.inlineCallbacks
def _setup_sent_transactions(self):
Expand Down Expand Up @@ -594,7 +606,7 @@ class Porter(object):
"select", r,
)

self._convert_rows("sent_transactions", headers, rows)
rows = self._convert_rows("sent_transactions", headers, rows)

inserted_rows = len(rows)
if inserted_rows:
Expand Down

0 comments on commit 9d419f4

Please sign in to comment.