Upgrade sqlalchemy to version 2

api/requirements.txt

@@ -1,7 +1,7 @@
 # Runtime requirements
 flask
 flask_cors
-sqlalchemy>=1.4.9,<2.0
+sqlalchemy~=2.0.0
 gunicorn
 rocky>=1,<2
 PyMySQL

api/src/core/auth.py

@@ -131,9 +131,8 @@ def password_reset(reset_token, unhashed_password):
     except ValueError as e:
         raise BadRequest(str(e))
 
-    try:
-        member = db_session.query(Member).get(password_reset_token.member_id)
-    except NoResultFound:
+    member = db_session.get(Member, password_reset_token.member_id)
+    if member is None:
         raise InternalServerError(log=f"No member with id {password_reset_token.member_id} found, this is a bug.")
 
     member.password = hashed_password

api/src/core/models.py

@@ -1,7 +1,6 @@
 from service.db import db_session
 from sqlalchemy import Column, DateTime, Integer, String, Text, func, text
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import configure_mappers
+from sqlalchemy.orm import configure_mappers, declarative_base
 
 Base = declarative_base()
 
@@ -46,19 +45,21 @@ class Login:
 
     @staticmethod
     def register_login_failed(ip):
-        db_session.execute("INSERT INTO login (success, ip) VALUES (0, :ip)", {"ip": ip})
+        db_session.execute(text("INSERT INTO login (success, ip) VALUES (0, :ip)"), {"ip": ip})
 
     @staticmethod
     def register_login_success(ip, user_id):
         db_session.execute(
-            "INSERT INTO login (success, user_id, ip) VALUES (1, :user_id, :ip)", {"user_id": user_id, "ip": ip}
+            text("INSERT INTO login (success, user_id, ip) VALUES (1, :user_id, :ip)"), {"user_id": user_id, "ip": ip}
         )
 
     @staticmethod
     def get_failed_login_count(ip):
         (count,) = db_session.execute(
-            "SELECT count(1) FROM login"
-            "   WHERE ip = :ip AND NOT success AND date >= DATE_SUB(NOW(), INTERVAL 1 HOUR)",
+            text(
+                "SELECT count(1) FROM login"
+                "   WHERE ip = :ip AND NOT success AND date >= DATE_SUB(NOW(), INTERVAL 1 HOUR)"
+            ),
             {"ip": ip},
         ).fetchone()
         return count

api/src/firstrun.py

@@ -134,7 +134,7 @@ def get_password():
             member_id = member["member_id"]
 
             logger.info(f"Adding new member {member_id} to admin group.")
-            admins.members.append(db_session.query(Member).get(member_id))
+            admins.members.append(db_session.get(Member, member_id))
             db_session.commit()
             break
         except Exception as e:

api/src/init_db.py

@@ -11,14 +11,15 @@
 from rocky.process import log_exception
 from service.config import get_mysql_config
 from service.db import create_mysql_engine
+from sqlalchemy import text
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
 
 
 def clear_permission_cache(session_factory):
     """Clear permisssion cache as a part of every db_init/restart."""
     with closing(session_factory()) as session:
-        session.execute("UPDATE access_tokens SET permissions = NULL")
+        session.execute(text("UPDATE access_tokens SET permissions = NULL"))
         session.commit()
 
 

api/src/member/member.py

@@ -37,7 +37,7 @@ def send_access_token_email(redirect, user_identification, ip, browser):
 
 
 def send_updated_member_info_email(member_id: int, msg_swe: str, msg_en: str):
-    member = db_session.query(Member).get(member_id)
+    member = db_session.get(Member, member_id)
 
     logger.info(
         f"sending email about updated personal information to member_id {member.member_id} with message {msg_en=},"
@@ -56,7 +56,7 @@ def send_updated_member_info_email(member_id: int, msg_swe: str, msg_en: str):
 
 
 def set_pin_code(member_id: int, pin_code: str):
-    member: Member = db_session.query(Member).get(member_id)
+    member: Member = db_session.get(Member, member_id)
     member.pin_code = pin_code
 
     try:

api/src/member/views.py

@@ -40,7 +40,7 @@ def current_member():
 
     # Expose if the member has a password set, but not what the password is (not even the hash)
     assert m is not None
-    m2 = db_session.query(Member).get(g.user_id)
+    m2 = db_session.get(Member, g.user_id)
     assert m2 is not None
     m["has_password"] = m2.password is not None
 

api/src/membership/member_entity.py

@@ -5,6 +5,7 @@
 from pymysql.constants.ER import DUP_ENTRY
 from service.db import db_session
 from service.entity import Entity
+from sqlalchemy import text
 from sqlalchemy.exc import IntegrityError
 
 from membership.member_auth import check_and_hash_password
@@ -46,7 +47,7 @@ def create(self, data=None, commit=True):
                 with db_session.begin_nested():
                     data = data.copy()
                     (max_member_number,) = db_session.execute(
-                        "SELECT COALESCE(MAX(member_number), 999) FROM membership_members"
+                        text("SELECT COALESCE(MAX(member_number), 999) FROM membership_members")
                     ).fetchone()
                     data["member_number"] = max_member_number + offset
                     # We must not commit here, because that will end our transaction, and the to_obj call will fail

api/src/membership/models.py

@@ -17,10 +17,8 @@
     Text,
     func,
     select,
-    text,
 )
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import column_property, configure_mappers, relationship, validates
+from sqlalchemy.orm import column_property, configure_mappers, declarative_base, relationship, validates
 
 Base = declarative_base()
 
@@ -73,7 +71,7 @@ class Member(Base):
     def validate_phone(self, key: Any, value: Optional[str]) -> Optional[str]:
         return normalise_phone_number(value)
 
-    groups = relationship("Group", secondary=member_group, back_populates="members")
+    groups = relationship("Group", secondary=member_group, back_populates="members", cascade_backrefs=False)
 
     def __repr__(self) -> str:
         return f"Member(member_id={self.member_id}, member_number={self.member_number}, email={self.email})"
@@ -99,9 +97,13 @@ class Group(Base):
     updated_at = Column(DateTime, server_default=func.now())
     deleted_at = Column(DateTime)
 
-    members = relationship("Member", secondary=member_group, lazy="dynamic", back_populates="groups")
+    members = relationship(
+        "Member", secondary=member_group, lazy="dynamic", back_populates="groups", cascade_backrefs=False
+    )
 
-    permissions = relationship("Permission", secondary=group_permission, back_populates="groups")
+    permissions = relationship(
+        "Permission", secondary=group_permission, back_populates="groups", cascade_backrefs=False
+    )
 
     def __repr__(self) -> str:
         return f"Group(group_id={self.group_id}, name={self.name})"
@@ -110,7 +112,7 @@ def __repr__(self) -> str:
 # Calculated property will be executed as a sub select for each groups, since it is not that many groups this will be
 # fine.
 Group.num_members = column_property(
-    select([func.count(member_group.columns.member_id)])
+    select(func.count(member_group.columns.member_id))
     .where(Group.group_id == member_group.columns.group_id)
     .scalar_subquery()
 )
@@ -125,7 +127,7 @@ class Permission(Base):
     updated_at = Column(DateTime, server_default=func.now())
     deleted_at = Column(DateTime)
 
-    groups = relationship("Group", secondary=group_permission, back_populates="permissions")
+    groups = relationship("Group", secondary=group_permission, back_populates="permissions", cascade_backrefs=False)
 
 
 class Key(Base):
@@ -139,7 +141,7 @@ class Key(Base):
     updated_at = Column(DateTime, server_default=func.now())
     deleted_at = Column(DateTime)
 
-    member = relationship(Member, backref="keys")
+    member = relationship(Member, backref="keys", cascade_backrefs=False)
 
     def __repr__(self) -> str:
         return f"Key(key_id={self.key_id}, tagid={self.tagid})"
@@ -162,7 +164,7 @@ class Span(Base):
     deleted_at = Column(DateTime)
     deletion_reason = Column(String(255))
 
-    member = relationship(Member, backref="spans")
+    member = relationship(Member, backref="spans", cascade_backrefs=False)
 
     def __repr__(self) -> str:
         return f"Span(span_id={self.span_id}, type={self.type}, enddate={self.enddate})"
@@ -188,7 +190,7 @@ class Box(Base):
     # last nag date for that member.
     last_nag_at = Column(DateTime, nullable=False)
 
-    member = relationship(Member, backref="boxes")
+    member = relationship(Member, backref="boxes", cascade_backrefs=False)
 
     def __repr__(self) -> str:
         return (
@@ -213,7 +215,7 @@ class PhoneNumberChangeRequest(Base):
     # When the request was made.
     timestamp = Column(DateTime, nullable=False)
 
-    member = relationship(Member, backref="change_phone_number_requests")
+    member = relationship(Member, backref="change_phone_number_requests", cascade_backrefs=False)
 
     @validates("phone")
     def validate_phone(self, key: Any, value: Optional[str]) -> Optional[str]:

api/src/messages/models.py

@@ -2,8 +2,7 @@
 
 from membership.models import Member
 from sqlalchemy import Column, Date, DateTime, Enum, ForeignKey, Integer, String, Text, func
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import configure_mappers, relationship
+from sqlalchemy.orm import configure_mappers, declarative_base, relationship
 
 Base = declarative_base()
 

api/src/migrate.py

@@ -7,7 +7,7 @@
 from os.path import dirname, exists, isdir, join
 
 from service.logging import logger
-from sqlalchemy import inspect
+from sqlalchemy import inspect, text
 
 Migration = namedtuple("Migration", "id,name")
 
@@ -26,53 +26,71 @@ def ensure_migrations_table(engine, session_factory):
     if "migrations" not in table_names:
         with closing(session_factory()) as session:
             logger.info("creating migrations table")
-            session.execute("ALTER DATABASE makeradmin CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci")
+            session.execute(text("ALTER DATABASE makeradmin CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci"))
             session.execute(
-                "CREATE TABLE migrations ("
-                "    id INTEGER NOT NULL,"
-                "    name VARCHAR(255) COLLATE utf8mb4_0900_ai_ci NOT NULL,"
-                "    applied_at DATETIME NOT NULL,"
-                "    PRIMARY KEY (id)"
-                ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"
+                text(
+                    "CREATE TABLE migrations ("
+                    "    id INTEGER NOT NULL,"
+                    "    name VARCHAR(255) COLLATE utf8mb4_0900_ai_ci NOT NULL,"
+                    "    applied_at DATETIME NOT NULL,"
+                    "    PRIMARY KEY (id)"
+                    ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci"
+                )
             )
             session.commit()
     elif "service" in {c["name"] for c in engine_inspect.get_columns("migrations")}:
         with closing(session_factory()) as session:
             logger.info("updating existing migrations table")
             session.execute(
-                "UPDATE migrations SET id=1, name='0001_initial_core'"
-                "     WHERE id=1 AND service='core' AND name='0001_initial'"
+                text(
+                    "UPDATE migrations SET id=1, name='0001_initial_core'"
+                    "     WHERE id=1 AND service='core' AND name='0001_initial'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=5, name='0005_remove_excessive_permissions'"
-                "    WHERE id=2 AND service='membership' AND name='0002_remove_excessive_permissions'"
+                text(
+                    "UPDATE migrations SET id=5, name='0005_remove_excessive_permissions'"
+                    "    WHERE id=2 AND service='membership' AND name='0002_remove_excessive_permissions'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=6, name='0006_add_box'"
-                "    WHERE id=3 AND service='membership' AND name='0003_add_box'"
+                text(
+                    "UPDATE migrations SET id=6, name='0006_add_box'"
+                    "    WHERE id=3 AND service='membership' AND name='0003_add_box'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=2, name='0002_initial_membership'"
-                "    WHERE id=1 AND service='membership' AND name='0001_initial'"
+                text(
+                    "UPDATE migrations SET id=2, name='0002_initial_membership'"
+                    "    WHERE id=1 AND service='membership' AND name='0001_initial'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=4, name='0004_initial_messages'"
-                "    WHERE id=1 AND service='messages' AND name='0001_initial'"
+                text(
+                    "UPDATE migrations SET id=4, name='0004_initial_messages'"
+                    "    WHERE id=1 AND service='messages' AND name='0001_initial'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=7, name='0007_rename_everything'"
-                "    WHERE id=2 AND service='messages' AND name='0002_rename_everything'"
+                text(
+                    "UPDATE migrations SET id=7, name='0007_rename_everything'"
+                    "    WHERE id=2 AND service='messages' AND name='0002_rename_everything'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=3, name='0003_initial_shop'"
-                "    WHERE id=1 AND service='shop' AND name='0001_initial'"
+                text(
+                    "UPDATE migrations SET id=3, name='0003_initial_shop'"
+                    "    WHERE id=1 AND service='shop' AND name='0001_initial'"
+                )
             )
             session.execute(
-                "UPDATE migrations SET id=8, name='0008_password_reset_token'"
-                "    WHERE id=2 AND service='core' AND name='0002_password_reset_token'"
+                text(
+                    "UPDATE migrations SET id=8, name='0008_password_reset_token'"
+                    "    WHERE id=2 AND service='core' AND name='0002_password_reset_token'"
+                )
             )
-            session.execute("ALTER TABLE migrations DROP PRIMARY KEY, ADD PRIMARY KEY(id)")
-            session.execute("ALTER TABLE migrations DROP COLUMN service")
+            session.execute(text("ALTER TABLE migrations DROP PRIMARY KEY, ADD PRIMARY KEY(id)"))
+            session.execute(text("ALTER TABLE migrations DROP COLUMN service"))
             session.commit()
 
 
@@ -99,7 +117,7 @@ def run_migrations(session_factory):
 
         migrations.sort(key=lambda m: m.id)
 
-        applied = {i: Migration(i, n) for i, n in session.execute("SELECT id, name FROM migrations ORDER BY ID")}
+        applied = {i: Migration(i, n) for i, n in session.execute(text("SELECT id, name FROM migrations ORDER BY ID"))}
         session.commit()
 
         logger.info(f"{len(migrations) - len(applied)} migrations to apply, {len(applied)} migrations already applied")
@@ -115,10 +133,10 @@ def run_migrations(session_factory):
                 logger.info(f"migrations, applying {migration.name}")
 
                 for sql in read_sql(join(migrations_dir, migration.name + ".sql")):
-                    session.execute(sql)
+                    session.execute(text(sql))
 
                 session.execute(
-                    "INSERT INTO migrations VALUES (:id, :name, :applied_at)",
+                    text("INSERT INTO migrations VALUES (:id, :name, :applied_at)"),
                     {
                         "id": migration.id,
                         "name": migration.name,

api/src/multiaccessy/invite.py

@@ -30,7 +30,7 @@ class LabaccessRequirements(Enum):
 
 
 def check_labaccess_requirements(member_id: int) -> LabaccessRequirements:
-    member = db_session.query(Member).get(member_id)
+    member = db_session.get(Member, member_id)
     if member is None:
         return LabaccessRequirements.MEMBER_MISSING
 

api/src/multiaccessy/models.py

@@ -1,7 +1,6 @@
 from membership.models import Member
 from sqlalchemy import Boolean, Column, DateTime, Enum, ForeignKey, Integer, Numeric, String, Text, func
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import configure_mappers, relationship
+from sqlalchemy.orm import configure_mappers, declarative_base, relationship
 
 Base = declarative_base()
 

api/src/multiaccessy/sync.py

@@ -22,7 +22,7 @@
 
 def get_wanted_access(today: date, member_id: Optional[int] = None) -> dict[PHONE, AccessyMember]:
     if member_id is not None:
-        member = db_session.query(Member).get(member_id)
+        member = db_session.get(Member, member_id)
         if member is None:
             raise Exception("Member does not exist")
         members = [member]
@@ -109,7 +109,7 @@ def sync(today: Optional[date] = None, member_id: Optional[int] = None) -> None:
     # If a specific member is given, sync only that member,
     # otherwise sync all members
     if member_id is not None:
-        member = db_session.query(Member).get(member_id)
+        member = db_session.get(Member, member_id)
         if member is None:
             raise Exception("Member does not exist")
         if member.phone is None:

api/src/pytest.ini

@@ -4,7 +4,6 @@ filterwarnings =
     ignore::DeprecationWarning:rocky.config
     # selenium not closed in some test, not sure why
     ignore:.*4444.*:ResourceWarning:selenium.webdriver.remote.remote_connection
-    ignore:.*not compatible with SQLAlchemy 2.0.*:DeprecationWarning:
     ignore::DeprecationWarning:stripe.*:
 log_cli = 1
 log_cli_level = INFO