feat: move default MySQL charset to utf8mb4

src/DIRAC/Core/Utilities/MySQL.py

@@ -896,7 +896,7 @@ def _createTables(self, tableDict, force=False):
               index is the list of fields to be indexed. This indexes will declared
               unique.
             "Engine": use the given DB engine, InnoDB is the default if not present.
-            "Charset": use the given character set. Default is latin1
+            "Charset": use the given character set. Default is utf8mb4
           force:
             if True, requested tables are DROP if they exist.
             if False, returned with S_ERROR if table exist.
@@ -1013,7 +1013,7 @@ def _createTables(self, tableDict, force=False):
                         )
 
                 engine = thisTable.get("Engine", "InnoDB")
-                charset = thisTable.get("Charset", "latin1")
+                charset = thisTable.get("Charset", "utf8mb4")
 
                 cmd = "CREATE TABLE `{}` (\n{}\n) ENGINE={} DEFAULT CHARSET={}".format(
                     table,

src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py

@@ -12,7 +12,7 @@ class DatasetManager:
     _tables["FC_MetaDatasets"] = {
         "Fields": {
             "DatasetID": "INT AUTO_INCREMENT",
-            "DatasetName": "VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL",
+            "DatasetName": "VARCHAR(128) CHARACTER SET utfmb4 COLLATE utf8mb4_bin NOT NULL",
             "MetaQuery": "VARCHAR(512)",
             "DirID": "INT NOT NULL DEFAULT 0",
             "TotalSize": "BIGINT UNSIGNED NOT NULL",

src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql

@@ -23,7 +23,7 @@ CREATE TABLE FC_Files(
     UID SMALLINT UNSIGNED NOT NULL,
     GID TINYINT UNSIGNED NOT NULL,
     Status SMALLINT UNSIGNED NOT NULL,
-    FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     INDEX (DirID),
     INDEX (UID,GID),
     INDEX (Status),
@@ -119,7 +119,7 @@ CREATE TABLE FC_Users (
 
 CREATE TABLE FC_StorageElements (
     SEID INTEGER AUTO_INCREMENT PRIMARY KEY,
-    SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     AliasName VARCHAR(127) DEFAULT '',
     UNIQUE KEY (SEName)
 ) ENGINE = INNODB;
@@ -159,7 +159,7 @@ CREATE TABLE FC_DirectoryInfo (
 
 CREATE TABLE FC_DirMeta (
     DirID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (DirID,MetaKey)
 ) ENGINE = INNODB;
@@ -168,7 +168,7 @@ CREATE TABLE FC_DirMeta (
 
 CREATE TABLE FC_FileMeta (
     FileID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (FileID,MetaKey)
 ) ENGINE = INNODB;
@@ -177,7 +177,7 @@ CREATE TABLE FC_FileMeta (
 
 CREATE TABLE FC_DirectoryTree (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(1024) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  INDEX (Parent),
  INDEX (DirName)
@@ -187,7 +187,7 @@ CREATE TABLE FC_DirectoryTree (
 
 CREATE TABLE FC_DirectoryTreeM (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  Level INT NOT NULL,
  INDEX (Level),
@@ -199,7 +199,7 @@ CREATE TABLE FC_DirectoryTreeM (
 
 CREATE TABLE FC_DirectoryLevelTree (
  DirID INT AUTO_INCREMENT PRIMARY KEY,
- DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
  Parent INT NOT NULL DEFAULT 0,
  Level INT NOT NULL,
  LPATH1 INT NOT NULL DEFAULT 0,
@@ -240,15 +240,15 @@ CREATE TABLE FC_DirectoryUsage(
 
 CREATE TABLE FC_MetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
 -- ------------------------------------------------------------------------------
 
 CREATE TABLE FC_FileMetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 

src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql

@@ -36,7 +36,7 @@ INSERT INTO FC_Statuses (StatusID, Status) values (1, 'FakeStatus');
 
 CREATE TABLE FC_StorageElements (
     SEID INTEGER AUTO_INCREMENT,
-    SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
     AliasName VARCHAR(127) DEFAULT '',
 
     PRIMARY KEY (SEID),
@@ -79,7 +79,7 @@ INSERT INTO FC_Users (UID, UserName) values (1, 'root');
 --
 -- create table FC_DirectoryList (
 --   DirID INT NOT NULL AUTO_INCREMENT,
---   Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+--   Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 --
 --   PRIMARY KEY (DirID),
 --
@@ -116,7 +116,7 @@ create table FC_DirectoryList (
   ModificationDate DATETIME,
   Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775,
   Status INTEGER NOT NULL DEFAULT 0,
-  Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 
   PRIMARY KEY (DirID),
   FOREIGN KEY (UID) REFERENCES FC_Users(UID),
@@ -158,7 +158,7 @@ CREATE TABLE FC_Files(
     Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775,
     ChecksumType ENUM('Adler32','MD5'),
     Checksum VARCHAR(32),
-    FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+    FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
 
     PRIMARY KEY (FileID),
     FOREIGN KEY (DirID) REFERENCES FC_DirectoryList(DirID) ON DELETE CASCADE,
@@ -223,7 +223,7 @@ CREATE TABLE FC_DirectoryUsage(
 
 CREATE TABLE FC_DirMeta (
     DirID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (DirID,MetaKey)
 ) ENGINE = INNODB;
@@ -232,7 +232,7 @@ CREATE TABLE FC_DirMeta (
 
 CREATE TABLE FC_FileMeta (
     FileID INTEGER NOT NULL,
-    MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname',
+    MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname',
     MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname',
     PRIMARY KEY (FileID,MetaKey)
 ) ENGINE = INNODB;
@@ -242,15 +242,15 @@ CREATE TABLE FC_FileMeta (
 
 CREATE TABLE FC_MetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
 -- ------------------------------------------------------------------------------
 
 CREATE TABLE FC_FileMetaFields (
   MetaID INT AUTO_INCREMENT PRIMARY KEY,
-  MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaType VARCHAR(128) NOT NULL
 ) ENGINE = INNODB;
 
@@ -280,7 +280,7 @@ CREATE TABLE FC_FileAncestors (
 
 CREATE TABLE FC_MetaDatasets (
   DatasetID INT AUTO_INCREMENT,
-  DatasetName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+  DatasetName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
   MetaQuery VARCHAR(512),
   DirID INT NOT NULL DEFAULT 0,
   TotalSize BIGINT UNSIGNED NOT NULL,

src/DIRAC/FrameworkSystem/DB/AuthDB.py

@@ -22,7 +22,7 @@
 
 class RefreshToken(Model):
     __tablename__ = "RefreshToken"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
     jti = Column(String(255), nullable=False, primary_key=True)
     issued_at = Column(Integer, nullable=False, default=0)
     access_token = Column(Text, nullable=False)
@@ -31,15 +31,15 @@ class RefreshToken(Model):
 
 class JWK(Model):
     __tablename__ = "JWK"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
     kid = Column(String(255), unique=True, primary_key=True, nullable=False)
     key = Column(Text, nullable=False)
     expires_at = Column(Integer, nullable=False, default=0)
 
 
 class AuthSession(Model):
     __tablename__ = "AuthSession"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
     id = Column(String(255), unique=True, primary_key=True, nullable=False)
     uri = Column(String(255))
     state = Column(String(255))

src/DIRAC/FrameworkSystem/DB/InstalledComponentsDB.py

@@ -27,7 +27,7 @@ class Component(componentsBase):
     """
 
     __tablename__ = "Components"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     componentID = Column("ComponentID", Integer, primary_key=True)
     system = Column("DIRACSystem", String(32), nullable=False)
@@ -87,7 +87,7 @@ class Host(componentsBase):
     """
 
     __tablename__ = "Hosts"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     hostID = Column("HostID", Integer, primary_key=True)
     hostName = Column("HostName", String(32), nullable=False)
@@ -138,7 +138,7 @@ class InstalledComponent(componentsBase):
     """
 
     __tablename__ = "InstalledComponents"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     componentID = Column("ComponentID", Integer, ForeignKey("Components.ComponentID"), primary_key=True)
     hostID = Column("HostID", Integer, ForeignKey("Hosts.HostID"), primary_key=True)
@@ -217,7 +217,7 @@ class HostLogging(componentsBase):
     """
 
     __tablename__ = "HostLogging"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     hostName = Column("HostName", String(32), nullable=False, primary_key=True)
     # status

src/DIRAC/FrameworkSystem/DB/TokenDB.py

@@ -24,7 +24,7 @@ class Token(Model, OAuth2TokenMixin):
     """This class describes token fields"""
 
     __tablename__ = "Token"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
     # access_token too large for varchar(255)
     # 767 bytes is the stated prefix limitation for InnoDB tables in MySQL version 5.6
     # https://stackoverflow.com/questions/1827063/mysql-error-key-specification-without-a-key-length

src/DIRAC/ProductionSystem/DB/ProductionDB.sql

@@ -30,7 +30,7 @@ CREATE TABLE Productions(
     Status  CHAR(32) DEFAULT 'New',
     PRIMARY KEY(ProductionID),
     INDEX(ProductionName)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS ProductionSteps;
@@ -50,7 +50,7 @@ CREATE TABLE ProductionSteps(
     InsertedTime DATETIME,
     PRIMARY KEY(StepID),
     UNIQUE INDEX(StepID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS ProductionTransformations;
@@ -62,7 +62,7 @@ CREATE TABLE ProductionTransformations(
     PRIMARY KEY(ProductionID, TransformationID),
     UNIQUE INDEX(TransformationID),
     FOREIGN KEY(ProductionID) REFERENCES Productions(ProductionID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 -- -------------------------------------------------------------------------------
 DROP TABLE IF EXISTS ProductionTransformationLinks;
@@ -74,6 +74,6 @@ CREATE TABLE ProductionTransformationLinks(
     INDEX(TransformationID),
     FOREIGN KEY(ProductionID) REFERENCES Productions(ProductionID),
     FOREIGN KEY(TransformationID) REFERENCES ProductionTransformations(TransformationID)
-) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4;
 
 SET FOREIGN_KEY_CHECKS = 1;

src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py

@@ -45,7 +45,7 @@ class AccountingCache(rmsBase):
     """AccountingCache table"""
 
     __tablename__ = "AccountingCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     name = Column("Name", String(64), nullable=False, primary_key=True)
     plotname = Column("PlotName", String(64), nullable=False, primary_key=True)
@@ -88,7 +88,7 @@ class DowntimeCache(rmsBase):
     """DowntimeCache table"""
 
     __tablename__ = "DowntimeCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     downtimeid = Column("DowntimeID", String(127), nullable=False, primary_key=True)
     name = Column("Name", String(64), nullable=False)
@@ -150,7 +150,7 @@ class GGUSTicketsCache(rmsBase):
     """GGUSTicketsCache table"""
 
     __tablename__ = "GGUSTicketsCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     gocsite = Column("GocSite", String(64), nullable=False, primary_key=True)
     tickets = Column("Tickets", String(1024), nullable=False)
@@ -186,7 +186,7 @@ class JobCache(rmsBase):
     """JobCache table"""
 
     __tablename__ = "JobCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     site = Column("Site", String(64), nullable=False, primary_key=True)
     status = Column("Status", String(16), nullable=False)
@@ -219,7 +219,7 @@ class PilotCache(rmsBase):
     """PilotCache table"""
 
     __tablename__ = "PilotCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     site = Column("Site", String(64), nullable=False, primary_key=True)
     ce = Column("CE", String(64), nullable=False, primary_key=True)
@@ -261,7 +261,7 @@ class PolicyResult(rmsBase):
     """PolicyResult table"""
 
     __tablename__ = "PolicyResult"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     policyname = Column("PolicyName", String(64), nullable=False, primary_key=True)
     statustype = Column("StatusType", String(16), nullable=False, server_default="", primary_key=True)
@@ -320,7 +320,7 @@ class SpaceTokenOccupancyCache(rmsBase):
     """SpaceTokenOccupancyCache table"""
 
     __tablename__ = "SpaceTokenOccupancyCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     endpoint = Column("Endpoint", String(128), nullable=False, primary_key=True)
     token = Column("Token", String(64), nullable=False, primary_key=True)
@@ -358,7 +358,7 @@ class TransferCache(rmsBase):
     """TransferCache table"""
 
     __tablename__ = "TransferCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     sourcename = Column("SourceName", String(64), nullable=False, primary_key=True)
     destinationname = Column("DestinationName", String(64), nullable=False, primary_key=True)

src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py

@@ -50,7 +50,7 @@ class ResourceStatusCache(rssBase):
     """
 
     __tablename__ = "ResourceStatusCache"
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     id = Column("ID", BigInteger, nullable=False, autoincrement=True, primary_key=True)
     sitename = Column("SiteName", String(64), nullable=False)
@@ -89,7 +89,7 @@ class ElementStatusBase:
     Prototype for tables.
     """
 
-    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8"}
+    __table_args__ = {"mysql_engine": "InnoDB", "mysql_charset": "utf8mb4"}
 
     name = Column("Name", String(64), nullable=False, primary_key=True)
     statustype = Column("StatusType", String(128), nullable=False, server_default="all", primary_key=True)