duplicate entry with session.merge

[jean-yves...@coorpacademy.com]

I use sqlalchemy to perform to insert and upsert of rows. During certain requests for many tables I get an IntegrityError duplicate entry when using session.merge(obj).
I tried to find an answer on the web and on the IRC but I came back empty-handed. I'm using python 2.7, alembic 0.9.1 and PyMysql 0.7.9. I'm using a mysql dialect to communicate to a mysql RDS database from AWS.
The stack trace is:
```

(pymysql.err.IntegrityError) (1062, u"Duplicate entry 'q/FtMv4syu5QXenCXvx6ZGhbLSFYSbo6Kgv8sjuZDZwW4j1Zh3ZFHtey/aqVOe5j' for key 'PRIMARY'")

	Traceback (most recent call last): File "/var/task/streaming_lambda.py", line 71, in lambda_handler session.commit() File "/var/task/sqlalchemy/orm/session.py", line 874, in commit self.transaction.commit() File "/var/task/sqlalchemy/orm/session.py", line 461, in commit self._prepare_impl() File "/var/task/sqlalchemy/orm/session.py", line 441, in _prepare_impl self.session.flush() File "/var/task/sqlalchemy/orm/session.py", line 2137, in flush self._flush(objects) File "/var/task/sqlalchemy/orm/session.py", line 2257, in _flush transaction.rollback(_capture_exception=True) File "/var/task/sqlalchemy/util/langhelpers.py", line 60, in __exit__ compat.reraise(exc_type, exc_value, exc_tb) File "/var/task/sqlalchemy/orm/session.py", line 2221, in _flush flush_context.execute() File "/var/task/sqlalchemy/orm/unitofwork.py", line 389, in execute rec.execute(self) File "/var/task/sqlalchemy/orm/unitofwork.py", line 548, in execute uow File "/var/task/sqlalchemy/orm/persistence.py", line 181, in save_obj mapper, table, insert) File "/var/task/sqlalchemy/orm/persistence.py", line 799, in _emit_insert_statements execute(statement, multiparams) File "/var/task/sqlalchemy/engine/base.py", line 945, in execute return meth(self, multiparams, params) File "/var/task/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection return connection._execute_clauseelement(self, multiparams, params) File "/var/task/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement compiled_sql, distilled_params File "/var/task/sqlalchemy/engine/base.py", line 1189, in _execute_context context) File "/var/task/sqlalchemy/engine/base.py", line 1393, in _handle_dbapi_exception exc_info File "/var/task/sqlalchemy/util/compat.py", line 202, in raise_from_cause reraise(type(exception), exception, tb=exc_tb, cause=cause) File "/var/task/sqlalchemy/engine/base.py", line 1182, in _execute_context context) File "/var/task/sqlalchemy/engine/default.py", line 469, in do_execute cursor.execute(statement, parameters) File "/var/task/pymysql/cursors.py", line 166, in execute result = self._query(query) File "/var/task/pymysql/cursors.py", line 322, in _query conn.query(q) File "/var/task/pymysql/connections.py", line 835, in query self._affected_rows = self._read_query_result(unbuffered=unbuffered) File "/var/task/pymysql/connections.py", line 1019, in _read_query_result result.read() File "/var/task/pymysql/connections.py", line 1302, in read first_packet = self.connection._read_packet() File "/var/task/pymysql/connections.py", line 981, in _read_packet packet.check_error() File "/var/task/pymysql/connections.py", line 393, in check_error err.raise_mysql_exception(self._data) File "/var/task/pymysql/err.py", line 107, in raise_mysql_exception raise errorclass(errno, errval) IntegrityError: (pymysql.err.IntegrityError) (1062, u"Duplicate entry 'q/FtMv4syu5QXenCXvx6ZGhbLSFYSbo6Kgv8sjuZDZwW4j1Zh3ZFHtey/aqVOe5j' for key 'PRIMARY'") [SQL: u'INSERT INTO user_table (_id, collection_name, platform, created_at, updated_at, deleted, account_complete, population_current, state, stars, language, age_range, private_account, user_credit, new_user, anonymous) VALUES (%(_id)s, %(collection_name)s, %(platform)s, %(created_at)s, %(updated_at)s, %(deleted)s, %(account_complete)s, %(population_current)s, %(state)s, %(stars)s, %(language)s, %(age_range)s, %(private_account)s, %(user_credit)s, %(new_user)s, %(anonymous)s)'] [parameters: {'user_credit': 0, 'age_range': None, 'anonymous': 0, 'language': None, 'new_user': 1, 'collection_name': None, 'created_at': datetime.datetime(2000, 1, 1, 0, 0), 'deleted': 0, 'account_complete': None, 'updated_at': None, 'platform': 'undefined', 'state': None, 'stars': None, '_id': 'q/FtMv4syu5QXenCXvx6ZGhbLSFYSbo6Kgv8sjuZDZwW4j1Zh3ZFHtey/aqVOe5jmS3lKMKns0LkrC+uZEifDHg32wqWG6Efm+h4ofptf1kZSKIp54yj243Av+lQFSrQOJd5SeOcn178SodEAmfEEpRhzy3S9NW6RO1e64zpxoY=', 'private_account': 0, 'population_current': 'undefined'}]
	2017/04/05/[$LATEST]4a80f975904746559d2ae8dd180638ed``` Jean-Yves PASQUIER

[Yaniv Greenberg]

I had the same problem, I'm not sure about your code, but for me, the source of the problem was that I tried to merge two different objects that had the same primary key, so the merge supposedly worked, but when I commited, I got the same error you did.

Tp solve it I just kept the items I wanted to merge in a dict (with the primary keys as the dict keys), and just after I finished preparing all the objects, I merged and then commited them all.

Hope that helps (if it's even still relevant), 

but it might help someone else with the same problem.