Emap using MS Sql Server as a target (proof of concept)

core/core-config-envs.EXAMPLE

@@ -9,6 +9,7 @@ UDS_JDBC_URL=jdbc:postgresql://host.docker.internal:5432/informdb
 UDS_SCHEMA=public
 UDS_USERNAME=postgres
 UDS_PASSWORD=postgres
+UDS_HIBERNATE_DIALECT=
 SPRING_RABBITMQ_HOST=rabbitmq
 SPRING_RABBITMQ_PORT=5672
 SPRING_RABBITMQ_USERNAME=emap

core/docker-compose.fakeuds.yml

@@ -12,6 +12,25 @@ services:
     restart: unless-stopped
     ports:
       - "${FAKEUDS_PORT}:5432"
+    profiles:
+      - fakeuds-pg
+  fakeuds-mssql:
+    image: mcr.microsoft.com/mssql/server:2019-CU32-ubuntu-20.04
+    env_file:
+      - ../../config/fakeuds-mssql-config-envs
+    environment:
+      ACCEPT_EULA: "Y"
+      MSSQL_PID: "Express"
+      TZ: "Europe/London"
+    volumes:
+      - mssql-data-fakeuds:/var/opt/mssql
+    restart: unless-stopped
+    ports:
+      - "${FAKEUDS_MSSQL_PORT}:1433"
+    profiles:
+      - fakeuds-mssql
+
 
 volumes:
   postgres-data-fakeuds:
+  mssql-data-fakeuds:

core/fakeuds-mssql-config-envs.EXAMPLE

@@ -0,0 +1 @@
+MSSQL_SA_PASSWORD=

core/pom.xml

@@ -85,6 +85,11 @@
       <artifactId>commons-lang3</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.microsoft.sqlserver</groupId>
+      <artifactId>mssql-jdbc</artifactId>
+    </dependency>
+
 
     <dependency>
       <groupId>org.postgresql</groupId>

core/src/main/java/uk/ac/ucl/rits/inform/datasinks/emapstar/repos/IdsEffectLogging.java

@@ -1,11 +1,11 @@
 package uk.ac.ucl.rits.inform.datasinks.emapstar.repos;
 
-import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Index;
+import javax.persistence.Lob;
 import javax.persistence.Table;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -35,9 +35,9 @@ public class IdsEffectLogging {
     private Long processMessageDuration;
     private Boolean error;
     private String messageType;
-    @Column(columnDefinition = "text")
+    @Lob
     private String message;
-    @Column(columnDefinition = "text")
+    @Lob
     private String stackTrace;
 
     /**

core/src/main/resources/application.properties

@@ -2,11 +2,11 @@ spring.datasource.url=${UDS_JDBC_URL}
 spring.datasource.username=${UDS_USERNAME}
 spring.datasource.password=${UDS_PASSWORD}
 
-spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.PostgreSQLDialect
+spring.jpa.properties.hibernate.dialect = ${UDS_HIBERNATE_DIALECT}
 spring.jpa.properties.hibernate.default_schema=${UDS_SCHEMA}
 spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults = false
 
-spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect
+spring.jpa.database-platform=${UDS_HIBERNATE_DIALECT}
 spring.jpa.hibernate.ddl-auto = update
 spring.jpa.show_sql=false
 spring.datasource.hikari.maximum-pool-size=2

core/src/test/resources/application-test.properties

@@ -2,9 +2,9 @@ spring.datasource.url=jdbc:h2:mem:informdb
 spring.datasource.username=
 spring.datasource.password=
 
-spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.H2Dialect
+spring.jpa.properties.hibernate.dialect = uk.ac.ucl.rits.inform.informdb.H2EmapDialect
 spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults = false
-spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
+spring.jpa.database-platform=uk.ac.ucl.rits.inform.informdb.H2EmapDialect
 
 #spring.jpa.show-sql=true
 

emap-setup/emap_runner/docker/docker_runner.py

@@ -37,6 +37,7 @@ def __init__(self,
         self.enable_waveform = first_not_none(enable_waveform, self.config.get("waveform", "enable_waveform"))
         self.use_fake_waveform = first_not_none(use_fake_waveform, self.config.get("waveform", "enable_waveform_generator"))
         self.use_fake_uds = first_not_none(use_fake_uds, self.config.get("fake_uds", "enable_fake_uds"))
+        self.use_fake_mssql = self.config.get("fake_mssql", "enable_fake_mssql")
 
     def run(
         self,
@@ -55,7 +56,7 @@ def run(
 
         self._check_paths_exist()
 
-        cmd = self.base_docker_compose_command.split()
+        cmd = self.base_docker_compose_command
         for arg in docker_compose_args:
             cmd += [x.strip('"') for x in arg.split()]
 
@@ -84,15 +85,30 @@ def run(
         return None
 
     @property
-    def base_docker_compose_command(self) -> str:
-        return (
-            "docker compose -f "
-            + " -f ".join(str(p) for p in self.docker_compose_paths)
-            + f' -p {self.config["EMAP_PROJECT_NAME"]} '
-        )
+    def base_docker_compose_command(self) -> list[str]:
+        cmd = ["docker", "compose"]
+        for p in self.docker_compose_file_paths:
+            cmd.extend(["-f", str(p)])
+        for pr in self.docker_compose_profiles:
+            cmd.extend(["--profile", str(pr)])
+        cmd.extend(["-p", self.config["EMAP_PROJECT_NAME"]])
+        return cmd
+
+    @property
+    def docker_compose_profiles(self) -> List[str]:
+        """Required Docker Compose profiles"""
+        profiles = []
+        if self.use_fake_uds and self.use_fake_mssql:
+            raise RuntimeError("cannot have both fake postgres and fake MSSQL")
+        elif self.use_fake_mssql:
+            profiles.append("fakeuds-mssql")
+        elif self.use_fake_uds:
+            profiles.append("fakeuds-pg")
+
+        return profiles
 
     @property
-    def docker_compose_paths(self) -> List[Path]:
+    def docker_compose_file_paths(self) -> List[Path]:
         """Paths of all the required docker-compose yamls"""
 
         paths = [
@@ -102,7 +118,7 @@ def docker_compose_paths(self) -> List[Path]:
         # Fakes are for testing only. Waveform is a real feature that is currently off
         # by default, except for the waveform generator which is for testing waveform
         # data only.
-        if self.use_fake_uds:
+        if self.use_fake_uds or self.use_fake_mssql:
             paths.append(Path(self.emap_dir, "core", "docker-compose.fakeuds.yml"))
         if self.enable_waveform:
             paths.append(Path(self.emap_dir, "waveform-reader", "docker-compose.yml"))
@@ -145,7 +161,7 @@ def setup_glowroot_password(self) -> None:
     def _check_paths_exist(self) -> None:
         """Ensure all the docker compose files exist"""
 
-        paths = self.docker_compose_paths
+        paths = self.docker_compose_file_paths
 
         if not all(path.exists() for path in paths):
             _paths_str = "\n".join(str(p) for p in paths)
@@ -159,7 +175,7 @@ def _check_paths_exist(self) -> None:
     @staticmethod
     def _all_global_environment_variables() -> dict:
         """Dictionary of all global variables present in
-        config/global-config-envs added to the currently set env vars"""
+        config/global-config-envs or config/*-build-args added to the currently set env vars"""
 
         config_dir_path = Path(Path.cwd(), "config")
 
@@ -171,10 +187,20 @@ def _all_global_environment_variables() -> dict:
         env_vars = os.environ.copy()
 
         for item in config_dir_path.iterdir():
-            # only necessary to read the global config variables; rest will be
-            # pulled through containers directly
-            if item.is_file() and item.stem == "global-config-envs":
-                env_vars.update(EnvironmentFile(item).environment_variables)
+            item: Path
+            # only necessary to read the global config variables and variables that become build args;
+            # rest will be pulled through containers directly
+            if item.is_file() and (item.stem == "global-config-envs" or item.stem.endswith('-build-args')):
+                new_variables = EnvironmentFile(item).environment_variables
+                # Can't see an easy way of keeping the build args separate per image, so just put them all
+                # in the environment and warn here if any variables conflict.
+                for nv in new_variables.keys():
+                    if nv in env_vars:
+                        logger.warning(
+                            "Conflicting build time variables:\n%s=%s [from %s] overwriting:\n%s=%s",
+                            nv, new_variables[nv], item.stem, nv, env_vars[nv]
+                        )
+                env_vars.update(new_variables)
 
         return env_vars
 

emap-setup/emap_runner/global_config.py

@@ -22,6 +22,7 @@ class GlobalConfiguration(dict):
         "glowroot",
         "common",
         "fake_uds",
+        "fake_mssql",
         "waveform",
         "monitoring",
     )

emap-setup/emap_runner/setup/repos.py

@@ -159,7 +159,7 @@ def environment_files(self) -> List[EnvironmentFile]:
 
     @staticmethod
     def _add_env_to_files(path: Path, files: List[EnvironmentFile]):
-        if path.is_file() and str(path).endswith("-envs.EXAMPLE"):
+        if path.is_file() and str(path).endswith(("-envs.EXAMPLE", "-build-args.EXAMPLE")):
             files.append(EnvironmentFile.from_example_file(path))
 
     @staticmethod

emap-setup/global-configuration-EXAMPLE.yaml

@@ -80,24 +80,53 @@ dates:
   end:
 
 # Configurations for the UDS.
-uds:
+# For postgres
+postgres-uds: &postgres-uds
   UDS_JDBC_URL: jdbc:postgresql://uds_db:5432/uds
+  # dialect to use for core informdb tables
+  UDS_HIBERNATE_DIALECT: uk.ac.ucl.rits.inform.informdb.PostgreSQLEmapDialect
+  # profile name to use for liquibase or other places where there is an opportunity to compute the correct value
+  # possible values: sqlserver, postgres
+  UDS_HIBERNATE_DIALECT_PROFILE: postgres
   UDS_SCHEMA: schemaname
   UDS_USERNAME: schemauser
   UDS_PASSWORD: schemapw
 
+# For SQL Server
+sqlserver-uds: &sqlserver-uds
+  UDS_JDBC_URL: jdbc:sqlserver://fakeuds-mssql:1433;Encrypt=false;TrustServerCertificate=true
+  # dialect to use for core informdb tables
+  UDS_HIBERNATE_DIALECT: uk.ac.ucl.rits.inform.informdb.SQLServer2016EmapDialect
+  # profile name to use for liquibase or other places where there is an opportunity to compute the correct value
+  UDS_HIBERNATE_DIALECT_PROFILE: sqlserver
+  UDS_SCHEMA: dbo
+  UDS_USERNAME: sa
+  UDS_PASSWORD: EXAMPLE_VALUE
+
+
+# Switch the actual config between Postgres and SQL Server here
+uds: *postgres-uds
+
 # These are the configurations for glowroot (a Java application performance
 # management system).
 glowroot:
   DOMAIN: glowroot_domain
   GLOWROOT_USERNAME: glowrootuser
   GLOWROOT_PASSWORD: glowrootpw
   GLOWROOT_ADMIN_PORT: 4000
+  MAX_HEAP_SIZE: 4G
+  HEAP_NEWSIZE: 800M
+
 
 # For testing outside the GAE, you can enable a fake UDS
 fake_uds:
   enable_fake_uds: false
 
+# Emap in MSSQL
+fake_mssql:
+  enable_fake_mssql: false
+  MSSQL_SA_PASSWORD: your_password
+
 # config related to waveform data ingress
 waveform:
   enable_waveform: false

emap-setup/tests/data/test-global-configuration-only-docs.yaml

@@ -11,6 +11,8 @@ repositories:
 # For testing outside the GAE, you can enable a fake UDS
 fake_uds:
   enable_fake_uds: false
+fake_mssql:
+  enable_fake_mssql: false
 # config related to waveform data ingress
 waveform:
   enable_waveform: false

emap-setup/tests/data/test-global-configuration-onlyhl7.yaml

@@ -43,6 +43,9 @@ uds:
 fake_uds:
   enable_fake_uds: false
 
+fake_mssql:
+  enable_fake_mssql: false
+
 # config related to waveform data ingress
 waveform:
   enable_waveform: false

emap-setup/tests/data/test-global-configuration.yaml

@@ -51,6 +51,9 @@ common:
 fake_uds:
   enable_fake_uds: false
 
+fake_mssql:
+  enable_fake_mssql: false
+
 # config related to waveform data ingress
 waveform:
   enable_waveform: false

emap-star/README.md

@@ -73,7 +73,8 @@ relational database. We have chosen a relational structure for ease of use and e
   ```
 - Timestamps (date and times) are timezone aware (and automated testing enforces this) and named in the form `<something>Datetime`
 - Dates should only have date information and be named in the form `<something>Date`
+- `timestamp with time zone` is Postgres-specific so should not be specified manually. Hibernate does the right thing in this case.
   ```java
-  @Column(columnDefinition = "timestamp with time zone")
-  private Instant admissionTime;
+  private Instant admissionDatetime;
+  private LocalDate addedDate;
   ```