New Docker development environment

.gitignore

@@ -1 +1,23 @@
-config/
+config/*
+!config/api_key.txt
+
+openmlsource/*
+!openmlsource/readme.md
+
+OpenML/
+OpenML2/
+OpenML-backup/
+
+openml.org/
+portml-website/
+portml-platform/
+
+storage/datastore/*
+storage/es/*
+storage/mysql/*
+storage/minio/*
+portml-website
+
+docker-compose.yml
+docker-compose-aws.yml
+docker-compose-python.yml

aws-readme.md

@@ -0,0 +1,84 @@
+# AWS instructions
+
+## Services used
+
+1.  Elastic Container Registry
+    - Used to upload and store images of different containers
+2. Elastic Container Service
+    - Running the containers as separate instances. Using Fargate clusters to run all container tasks in a single service
+3. Elastic File System
+    - Running storage for shared volumes (MySQL, ES, config, and processing data) storage.
+4. EC2 Load balancer
+    - Used to route traffic from PortML domain to correct instances on ECS
+5. CloudWatch
+    - Logging for ECS containers
+
+## Compiling AWS images
+
+
+## Uploading new images
+Once per session connect docker to AWS ECR:
+```
+aws ecr get-login-password --region [AWS_REGION] | docker login --username AWS --password-stdin [AWS_ACCOUNT_ID].dkr.ecr.[AWS_REGION].amazonaws.com
+```
+Fill in the AWS region (twice) and the AWS account ID
+
+Next tag your images to push:
+```
+docker tag [DOCKER_IMAGE_NAME] [AWS_ACCOUNT_ID].dkr.ecr.[AWS_REGION].amazonaws.com/[AWS_ECR_REPOSITORY_NAME]
+```
+When successful push the image to AWS ECR:
+```
+docker push [AWS_ACCOUNT_ID].dkr.ecr.[AWS_REGION].amazonaws.com/[AWS_ECR_REPOSITORY_NAME]
+```
+
+## Setting the task definitions on AWS ECS
+Copy ``.reactenv`` file in ``[FLASK directory]`` and rename to ``.reactenv_aws`` and set all the settings according to the AWS settings to be used
+
+Instead of the default ``docker compose`` up command use the following chained commands to make use of the defined AWS compose file: ``docker-compose-aws.yml`` 
+```
+docker-compose rm -f ; docker-compose pull ; docker-compose -f docker-compose.yml -f docker-compose-aws.yml up --build
+```
+
+## Running config commands
+
+```
+aws ecs execute-command --region [AWS_REGION] --cluster fargate-cluster --task [TASKID] --container website --command "php index.php cron init_local_env" --interactive
+```
+
+
+## Additional info
+Extra information regarding AWS previously posted into Slack
+
+### To update loadbalancers on service:
+```
+aws ecs update-service --service portml-dev-service --cluster fargate-cluster --cli-input-json file://D:\Work\PortML\load-v3.json
+```
+jsonContent:
+```
+{
+	"loadBalancers" :
+	[
+		{
+			"targetGroupArn": "arn:aws:elasticloadbalancing:eu-west-1:482280224187:targetgroup/portml-dev-website-new-tg/1771310010e11e59",
+			"containerName": "website_new",
+			"containerPort": 5000
+		},
+		{
+			"targetGroupArn": "arn:aws:elasticloadbalancing:eu-west-1:482280224187:targetgroup/portml-dev-elasticsearch-tg/1ff8abd71c2954a6",
+			"containerName": "elasticsearch",
+			"containerPort": 9200
+		},
+		{
+			"targetGroupArn": "arn:aws:elasticloadbalancing:eu-west-1:482280224187:targetgroup/portml-dev-website-tg/6171a39595c1fd72",
+			"containerName": "website",
+			"containerPort": 80
+		}
+	]
+}
+```
+
+### Run PhP my admin for AWS (if port is open)
+```
+docker run --name myadmin -d -e PMA_HOST=[IP of task] -p 8000:80 phpmyadmin
+```

config/api_key.txt

@@ -0,0 +1 @@
+0123456789abcdef0123456789abcdef

docker-compose-aws-template.yml

@@ -0,0 +1,47 @@
+# OpenML dev docker-compose
+# For local dev environment setup
+# *Insecure & Not for Production Setup*
+
+# Notes:
+# issues: Rui Quintino,https://github.com/rquintino
+# config folder is used to share some config keys after init between containers
+# $ needs escaping to $$ in docker_compose file, $api_key is then $$api_key
+# TODO: gollum not working, CORS issue, needs apache mapping, local gollum install on website container?
+# OpenML main repo is mapped as volume, not copied within the built images, for enable development
+
+version: '2'
+
+services:
+  # MySQL dbs
+  mysql:
+    image: mysql:5.7.25
+    container_name: mysql_test
+    platform: linux/x86_64
+    ports:
+      # <Port exposed> : < MySQL Port running inside container>
+      - '3307:3306'
+    build:
+      #Specify the build file
+      context: ./mysql
+      dockerfile: DockerfileAws
+    environment:
+      MYSQL_ROOT_PASSWORD: "password"
+      MYSQL_DATABASE: openml
+    volumes:
+      - ./storage/mysql:/storage
+
+  website_new:
+    build:
+      context: ./
+      dockerfile: ./website-new/DockerfileAws
+      args:
+        - sourceDirectory=openml.org
+    ports:
+      - "5000:5000"
+    links:
+      - mysql
+    depends_on:
+      - mysql
+      - elasticsearch
+    environment:
+      - PYTHONUNBUFFERED=1

docker-compose-template.yml

@@ -0,0 +1,139 @@
+# OpenML dev docker-compose
+# For local dev environment setup
+# *Insecure & Not for Production Setup*
+
+# Notes:
+# issues: Rui Quintino,https://github.com/rquintino
+# config folder is used to share some config keys after init between containers
+# $ needs escaping to $$ in docker_compose file, $api_key is then $$api_key
+# TODO: gollum not working, CORS issue, needs apache mapping, local gollum install on website container?
+# OpenML main repo is mapped as volume, not copied within the built images, for enable development
+
+version: '2'
+
+services:
+  # MySQL dbs
+  mysql:
+    image: mysql:5.7.25
+    container_name: mysql_test
+    platform: linux/x86_64
+    ports:
+      # <Port exposed> : < MySQL Port running inside container>
+      - '3307:3306'
+    build: ./mysql
+    environment:
+      MYSQL_ROOT_PASSWORD: "root_password"
+      MYSQL_DATABASE: openml
+    volumes:
+      - ./storage/mysql:/storage
+
+  #Flask + React website
+  website_new:
+    build:
+      context: ./
+      dockerfile: ./website-new/Dockerfile
+      args:
+        - sourceDirectory=openml.org
+    ports:
+      - "5000:5000"
+    links:
+      - mysql
+    depends_on:
+      - mysql
+      - elasticsearch
+    environment:
+      - PYTHONUNBUFFERED=1
+      - DATABASE_URI=mysql+pymysql://root:[fil-in PASSWORD]@[fill-in MYSQL_CONTAINER_NAME]:3306/[fill-in DATABASE_NAME]
+      - ELASTICSEARCH_SERVER=[ES_URL_SECURED]
+      - ELASTICSEARCH_CONTAINER_URL=[ES_URL]
+      - APP_SECRET_KEY=abcd
+      - JWT_SECRET_KEY=abcd
+      - SERVER_URL=http://website_new:5000/
+      - REDIRECT_URL=http://website_new:5000
+      - BACKEND_SERVER=http://website/api/v1/xml/
+      - BACKEND_BASE_URL=http://website/
+      - BACKEND_CONTAINER_URL=http://website/
+      - SEND_EMAIL=False
+      - DASHBOARD_USE_DOCKER_CONTAINER_NAME=FALSE
+      - DASHBOARD_PHP_CONTAINER_NAME=website
+      - AUTH_URL_PREFIX=/secure
+      - REQUIRE_CONFIRMED_REGISTERING=False
+      - EXTERNAL_WEBISTE_URL=http://localhost:5000/
+
+  # Elastic search indexes
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.8.2
+    build: ./elasticsearch
+    command: elasticsearch
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+    volumes:
+      - ./storage/es:/esdata
+
+  # Php website
+  website:
+    build:
+      context: ./
+      dockerfile: ./website/Dockerfile
+      args:
+        - sourceDirectory=OpenML
+    ports:
+      - 80:80
+    volumes:
+      - ./config:/openmlconfig
+      - ./storage/datastore:/datastore
+    depends_on:
+      - mysql
+      - elasticsearch
+    environment:
+      DB_NAME_OPENML: 'openml'
+      DB_HOST_OPENML: "mysql_test"
+      DB_USER_OPENML: 'username'
+      DB_PASS_OPENML: 'user_password'
+      DB_NAME_EXPDB: 'openml_expdb'
+      DB_HOST_EXPDB: "mysql_test"
+      DB_USER_EXPDB_READ: 'username'
+      DB_PASS_EXPDB_READ: 'user_password'
+      DB_USER_EXPDB_WRITE: 'username'
+      DB_PASS_EXPDB_WRITE: 'user_password'
+      ES_URL: 'elasticsearch:9200'
+      ES_PUBLIC_URL: 'localhost:9200'
+      BASE_URL: 'http://localhost/'
+      DATA_PATH: '/datastore/'
+      DIRECTORY_PATH: '/var/www/html/'
+      API_USERNAME: 'fill in'
+      API_KEY: 'fill in'
+
+  # MySQL admin UI (tool)
+  phpmyadmin:
+    image: phpmyadmin/phpmyadmin:latest
+    volumes:
+      - ./config:/openmlconfig
+    ports:
+      - 8080:80
+    environment:
+      - PMA_HOST=mysql
+    depends_on:
+      - mysql
+
+  # OpenML Java background workers
+  java:
+    build: ./java
+    image: amazoncorretto
+    volumes:
+      - ./config:/openmlconfig
+      - ./openmlsource/:/openmlsource
+    entrypoint: /entrypoint.sh
+
+  minio:
+    build: ./minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - ./storage/minio:/data
+    environment:
+      MINIO_ROOT_USER: FILL-IN
+      MINIO_ROOT_PASSWORD: FILL-IN
+    command: server --console-address ":9001" /data

elasticsearch/Dockerfile

@@ -1,4 +1,4 @@
-FROM elasticsearch
+FROM elasticsearch:6.8.2
 LABEL maintainer="Rui Quintino,https://github.com/rquintino"
 
 COPY ./elasticsearch.yml /usr/share/elasticsearch/config/

elasticsearch/elasticsearch.yml

@@ -8,4 +8,7 @@ http.host: 0.0.0.0
 http.cors.enabled: true
 http.cors.allow-origin: "*"
 http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE
-http.cors.allow-headers: "X-Requested-With,X-Auth-Token,Content-Type, Content-Length, Authorization"
+http.cors.allow-headers: X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
+
+#set data path
+path.data: /esdata

java/Dockerfile

@@ -0,0 +1,4 @@
+FROM amazoncorretto
+
+COPY ./entrypoint.sh /entrypoint.sh
+COPY ./entrypoint-aws.sh /entrypoint-aws.sh