diff --git a/README.md b/README.md index dbbaeef..e57916b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ python3 -m pip install -r requirements/common.txt The `terraform` cli package is required, unless you want to generate a project only locally. To install it we suggest to use the official [install guide](https://learn.hashicorp.com/tutorials/terraform/install-cli). +Should you opt for an S3-like object storage, you must install and launch MinIO Server's `mc` package as outlined in this [install guide](https://min.io/docs/minio/linux/index.html). + ## 🔑 Credentials (optional) ### 🦊 GitLab @@ -202,6 +204,16 @@ If you don't want DigitalOcean DNS configuration the following args are required | local | Docker Volume are used to store media | `--media-storage=local` | | none | Project have no media | `--media-storage=none` | +#### Local S3 storage + +For enabling a local S3-like object storage the following argument is needed: + +`--local-s3-storage` + +Disabled arg: + +`--no-local-s3-storage` + #### Redis For enabling redis integration the following arguments are needed: diff --git a/bootstrap/collector.py b/bootstrap/collector.py index 287a37e..2b1bd9e 100644 --- a/bootstrap/collector.py +++ b/bootstrap/collector.py @@ -61,6 +61,7 @@ class Collector: sentry_org: str | None = None sentry_url: str | None = None media_storage: str | None = None + local_s3_storage: bool | None = None gitlab_url: str | None = None gitlab_token: str | None = None gitlab_namespace_path: str | None = None @@ -89,6 +90,7 @@ def collect(self): self.set_sentry() self.set_gitlab() self.set_media_storage() + self.set_local_s3_storage() def set_project_slug(self): """Set the project slug option.""" @@ -286,6 +288,13 @@ def set_media_storage(self): type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False), ).lower() + def set_local_s3_storage(self): + """Set the local S3 storage option.""" + if "s3" in self.media_storage and self.local_s3_storage is None: + self.local_s3_storage = click.confirm( + warning("Do you want to use the local S3 storage?"), default=False + ) + def get_runner(self): """Get the bootstrap runner instance.""" return Runner( @@ -315,6 +324,7 @@ def get_runner(self): sentry_org=self.sentry_org, sentry_url=self.sentry_url, media_storage=self.media_storage, + local_s3_storage=self.local_s3_storage, use_redis=self.use_redis, gitlab_url=self.gitlab_url, gitlab_token=self.gitlab_token, diff --git a/bootstrap/runner.py b/bootstrap/runner.py index 96ea1ad..af1a36b 100644 --- a/bootstrap/runner.py +++ b/bootstrap/runner.py @@ -73,6 +73,7 @@ class Runner: sentry_url: str | None = None media_storage: str use_redis: bool = False + local_s3_storage: bool = False gitlab_url: str | None = None gitlab_namespace_path: str | None = None gitlab_token: str | None = None @@ -251,6 +252,7 @@ def init_service(self): "terraform_cloud_organization": self.terraform_cloud_organization, "tfvars": self.tfvars, "use_redis": self.use_redis and "true" or "false", + "local_s3_storage": self.local_s3_storage and "true" or "false", "use_vault": self.vault_url and "true" or "false", }, output_dir=self.output_dir, diff --git a/cookiecutter.json b/cookiecutter.json index 937ebdb..74f5e10 100755 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -10,6 +10,7 @@ "terraform_cloud_organization": "", "media_storage": ["digitalocean-s3", "other-s3", "local", "none"], "use_redis": "false", + "local_s3_storage": "false", "use_vault": "false", "environments_distribution": "1", "resources": { diff --git a/start.py b/start.py index e9c4d55..bb4eba3 100755 --- a/start.py +++ b/start.py @@ -62,6 +62,7 @@ "--media-storage", type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False), ) +@click.option("--local-s3-storage/--no-local-s3-storage", is_flag=True, default=None) @click.option("--use-redis/--no-redis", is_flag=True, default=None) @click.option("--gitlab-url") @click.option("--gitlab-token", envvar=GITLAB_TOKEN_ENV_VAR) diff --git a/{{cookiecutter.project_dirname}}/.env_template b/{{cookiecutter.project_dirname}}/.env_template index c3428fc..ad6d61d 100644 --- a/{{cookiecutter.project_dirname}}/.env_template +++ b/{{cookiecutter.project_dirname}}/.env_template @@ -3,7 +3,8 @@ CACHE_URL=locmem:// COMPOSE_FILE=docker-compose.yaml DATABASE_URL=postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }} DJANGO_ADMINS=admin,errors@example.org -DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }} +DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }}{% if "s3" in cookiecutter.media_storage %} +DJANGO_AWS_S3_URL=http://minio-admin:minio-admin@172.20.0.11:9000/{{ cookiecutter.project_slug }}{% endif %} DJANGO_CONFIGURATION=Local DJANGO_DEBUG=True DJANGO_DEFAULT_FROM_EMAIL=info@example.org diff --git a/{{cookiecutter.project_dirname}}/.gitlab-ci.yml b/{{cookiecutter.project_dirname}}/.gitlab-ci.yml index 68c4dad..b3ace2d 100644 --- a/{{cookiecutter.project_dirname}}/.gitlab-ci.yml +++ b/{{cookiecutter.project_dirname}}/.gitlab-ci.yml @@ -134,11 +134,15 @@ test: {{ cookiecutter.service_slug|upper }}_BUILD_TARGET: "test" {{ cookiecutter.service_slug|upper }}_IMAGE_NAME: "gitlabci_{{ cookiecutter.project_slug }}_{{ cookiecutter.service_slug }}" {{ cookiecutter.service_slug|upper }}_IMAGE_TAG: "${CI_JOB_NAME}-${CI_JOB_ID}" - COMPOSE_PROJECT_NAME: "${CI_PROJECT_PATH_SLUG}-${CI_JOB_NAME}-${CI_JOB_ID}" + COMPOSE_PROJECT_NAME: "${CI_PROJECT_PATH_SLUG}-${CI_JOB_NAME}-${CI_JOB_ID}"{% if cookiecutter.local_s3_storage == "true" %} + DJANGO_AWS_S3_URL: http://minio-admin:minio-admin@172.20.0.11:9000/{{ cookiecutter.project_slug }}{% endif %} script: - docker-compose build - docker-compose run --name ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME} {{ cookiecutter.service_slug }} - - docker cp ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME}:/app/htmlcov htmlcov + - docker cp ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME}:/app/htmlcov htmlcov{% if cookiecutter.local_s3_storage == "true" %} + - docker network create --subnet=172.20.0.0/16 custom + - docker network connect --ip 172.20.0.11 custom minio + - docker network connect --ip 172.20.0.12 custom postgres{% endif %} after_script: - docker-compose down -v coverage: '/^TOTAL.*\s+(\d+\%)$/' diff --git a/{{cookiecutter.project_dirname}}/Dockerfile b/{{cookiecutter.project_dirname}}/Dockerfile index 0158fef..87f9760 100644 --- a/{{cookiecutter.project_dirname}}/Dockerfile +++ b/{{cookiecutter.project_dirname}}/Dockerfile @@ -69,10 +69,11 @@ RUN apt-get update \ make \ openssh-client \ postgresql-client +RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc > /usr/bin/minio-client USER $APPUSER COPY --chown=$APPUSER ./requirements/local.txt requirements/local.txt RUN python3 -m pip install --user --no-cache-dir -r requirements/local.txt COPY --chown=$APPUSER . . RUN DJANGO_SECRET_KEY=build python3 -m manage collectstatic --clear --link --noinput -ENTRYPOINT ["./scripts/entrypoint.sh"] +ENTRYPOINT ["./scripts/entrypoint_local.sh"] CMD python3 -m manage runserver 0.0.0.0:${INTERNAL_SERVICE_PORT} diff --git a/{{cookiecutter.project_dirname}}/Makefile b/{{cookiecutter.project_dirname}}/Makefile index 421c27c..113d795 100644 --- a/{{cookiecutter.project_dirname}}/Makefile +++ b/{{cookiecutter.project_dirname}}/Makefile @@ -18,7 +18,12 @@ compilemessages: ## Django compilemessages .PHONY: coverage coverage: ## Run coverage - ./scripts/coverage.sh + ./scripts/coverage.sh{% if cookiecutter.local_s3_storage == "true" %} + +.PHONY: createbucket +createbucket: ## Create MinIO bucket + minio-client mb --quiet minio/{{ cookiecutter.project_slug }}; + minio-client anonymous set public minio/{{ cookiecutter.project_slug }};{% endif %} .PHONY: createsuperuser createsuperuser: ## Django createsuperuser diff --git a/{{cookiecutter.project_dirname}}/docker-compose.yaml b/{{cookiecutter.project_dirname}}/docker-compose.yaml index 75b3adb..e9d4a79 100644 --- a/{{cookiecutter.project_dirname}}/docker-compose.yaml +++ b/{{cookiecutter.project_dirname}}/docker-compose.yaml @@ -10,12 +10,15 @@ services: image: ${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_NAME:-{{ cookiecutter.project_slug }}_{{ cookiecutter.service_slug }}}:${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_TAG:-latest} depends_on: postgres: - condition: service_healthy + condition: service_healthy{% if cookiecutter.local_s3_storage == "true" %} + minio: + condition: service_healthy{% endif %} environment: - CACHE_URL - DATABASE_URL=${DATABASE_URL:-postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }}} - DJANGO_ADMINS - - DJANGO_ALLOWED_HOSTS + - DJANGO_ALLOWED_HOSTS{% if "s3" in cookiecutter.media_storage %} + - DJANGO_AWS_S3_URL{% endif %} - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Testing} - DJANGO_DEBUG - DJANGO_DEFAULT_FROM_EMAIL @@ -32,7 +35,10 @@ services: - PYTHONBREAKPOINT ports: - "${{ '{' }}{{ cookiecutter.service_slug|upper }}_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}:${INTERNAL_SERVICE_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}" - user: ${USER:-appuser} + user: ${USER:-appuser}{% if cookiecutter.local_s3_storage == "true" %} + networks: + custom: + ipv4_address: 172.20.0.10{% endif %} postgres: environment: @@ -46,7 +52,39 @@ services: retries: 30 image: postgres:14-bullseye volumes: - - pg_data:/var/lib/postgresql/data + - pg_data:/var/lib/postgresql/data{% if cookiecutter.local_s3_storage == "true" %} + networks: + custom: + ipv4_address: 172.20.0.11 + + minio: + command: minio server /var/lib/minio/data --console-address ":9001" + environment: + - MINIO_ENDPOINT=http://minio:9000 + - MINIO_BUCKET_NAME={{ cookiecutter.project_slug }} + - MINIO_ROOT_USER=minio-admin + - MINIO_ROOT_PASSWORD=minio-admin + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 3s + timeout: 3s + retries: 30 + image: minio/minio:RELEASE.2024-01-31T20-20-33Z + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio_data:/var/lib/minio/data + networks: + custom: + ipv4_address: 172.20.0.12{% endif %} volumes: - pg_data: {} + pg_data: {}{% if cookiecutter.local_s3_storage == "true" %} + minio_data: {} + +networks: + custom: + ipam: + config: + - subnet: 172.20.0.0/16{% endif %} diff --git a/{{cookiecutter.project_dirname}}/requirements/common.in b/{{cookiecutter.project_dirname}}/requirements/common.in index 8147469..ad27d67 100644 --- a/{{cookiecutter.project_dirname}}/requirements/common.in +++ b/{{cookiecutter.project_dirname}}/requirements/common.in @@ -1,3 +1,4 @@ -r base.in django-configurations[cache,database,email]~=2.5.0 -django~=5.0.0 +{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0 +{% endif %}django~=5.0.0 diff --git a/{{cookiecutter.project_dirname}}/requirements/remote.in b/{{cookiecutter.project_dirname}}/requirements/remote.in index 9efef0e..d32ad55 100644 --- a/{{cookiecutter.project_dirname}}/requirements/remote.in +++ b/{{cookiecutter.project_dirname}}/requirements/remote.in @@ -1,7 +1,6 @@ -r common.in argon2-cffi~=23.1.0 -{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0 -{% endif %}gunicorn~=21.2.0 +gunicorn~=21.2.0 {% if cookiecutter.use_redis == "true" %}redis~=5.0.0 {% endif %}sentry-sdk~=1.39.0 uvicorn[standard]~=0.25.0 diff --git a/{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh b/{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh new file mode 100755 index 0000000..b8e9643 --- /dev/null +++ b/{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euo pipefail + +./entrypoint.sh +minio-client alias set minio http://minio:9000 minio-admin minio-admin; +minio-client mb --quiet --ignore-existing minio/{{ cookiecutter.project_slug }}; +minio-client anonymous set download minio/{{ cookiecutter.project_slug }}; +exec "${@}" diff --git a/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py b/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py index b6ff07c..5a1558f 100644 --- a/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py +++ b/{{cookiecutter.project_dirname}}/{{cookiecutter.django_settings_dirname}}/settings.py @@ -10,6 +10,7 @@ https://docs.djangoproject.com/en/stable/ref/settings/ """ +import re import string from copy import deepcopy from pathlib import Path @@ -142,6 +143,50 @@ class ProjectDefault(Configuration): # MEDIA_ROOT = BASE_DIR / "media"{% endif %} + {% if cookiecutter.local_s3_storage == "true" %}# Django Storages + # https://django-storages.readthedocs.io/en/latest/ + + AWS_S3_URL = values.Value() + + @property + def AWS_S3_URL_PARTS(self): + """Return the AWS S3 URL parts.""" + pattern = re.compile( + r"^(?Phttps?):\/\/(?P.+?):(?P.+?)" + r"@(?P.+?)(?::(?P\d*?))?\/(?P.+)$" + ) + return pattern.match(self.AWS_S3_URL).groupdict() + + @property + def AWS_STORAGE_BUCKET_NAME(self): + """Return the AWS storage bucket name.""" + return self.AWS_S3_URL_PARTS["bucket_name"] + + @property + def AWS_S3_ENDPOINT_URL(self): + """Return the AWS S3 endpoint URL.""" + return ( + f'{self.AWS_S3_URL_PARTS["scheme"]}://' + f'{self.AWS_S3_URL_PARTS["host"]}' + f':{self.AWS_S3_URL_PARTS["port"]}' + ) + + AWS_LOCATION = "" + + AWS_S3_FILE_OVERWRITE = False{% endif %} + + # Storage + # https://docs.djangoproject.com/en/stable/ref/files/storage/ + + STORAGES = { + "default": { + "BACKEND": "storages.backends.s3boto3.S3Boto3Storage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + }{% endif %} + # Email Settings # https://docs.djangoproject.com/en/stable/topics/email/ @@ -281,7 +326,7 @@ class Testing(ProjectDefault): # Cache URL # https://django-configurations.readthedocs.io/en/stable/values/ - CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}} + CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}{% if "s3" not in cookiecutter.media_storage %} # Storages # https://docs.djangoproject.com/en/stable/ref/settings/#std-setting-STORAGES @@ -293,7 +338,7 @@ class Testing(ProjectDefault): "staticfiles": { "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", }, - } + }{% endif %} # The MD5 based password hasher is much less secure but faster # https://docs.djangoproject.com/en/stable/topics/auth/passwords/ @@ -405,10 +450,10 @@ def STORAGES(self): # pragma: no cover """Return the storage settings.""" storages = deepcopy( ProjectDefault.STORAGES - ) # noqa{% if "s3" in cookiecutter.media_storage %} + ){% if "s3" in cookiecutter.media_storage and cookiecutter.local_s3_storage == "false" %} storages["default"][ "BACKEND" - ] = "storages.backends.s3boto3.S3Boto3Storage" # noqa{% endif %} + ] = "storages.backends.s3boto3.S3Boto3Storage"{% endif %} try: # WhiteNoise # http://whitenoise.evans.io/en/stable/django.html @@ -420,7 +465,39 @@ def STORAGES(self): # pragma: no cover storages["staticfiles"][ "BACKEND" ] = "whitenoise.storage.CompressedManifestStaticFilesStorage" - return storages + return storages{% if "s3" in cookiecutter.media_storage and cookiecutter.local_s3_storage == "false" %} + + # Django Storages + # https://django-storages.readthedocs.io/en/latest/ + + AWS_S3_URL = values.Value() + + @property + def AWS_S3_URL_PARTS(self): + """Return the AWS S3 URL parts.""" + pattern = re.compile( + r"^(?Phttps?):\/\/(?P.+?):(?P.+?)" + r"@(?P.+?)(?::(?P\d*?))?\/(?P.+)$" + ) + return pattern.match(self.AWS_S3_URL).groupdict() + + @property + def AWS_STORAGE_BUCKET_NAME(self): + """Return the AWS storage bucket name.""" + return self.AWS_S3_URL_PARTS["bucket_name"] + + @property + def AWS_S3_ENDPOINT_URL(self): + """Return the AWS S3 endpoint URL.""" + return ( + f'{self.AWS_S3_URL_PARTS["scheme"]}://' + f'{self.AWS_S3_URL_PARTS["host"]}' + f':{self.AWS_S3_URL_PARTS["port"]}' + ) + + AWS_LOCATION = "" + + AWS_S3_FILE_OVERWRITE = False{% endif %} # Sentry # https://sentry.io/for/django/ @@ -436,21 +513,10 @@ def STORAGES(self): # pragma: no cover sentry_sdk.init( integrations=[DjangoIntegration(), RedisIntegration()], send_default_pii=True, - ) # noqa{% else %} + ){% else %} from sentry_sdk.integrations.django import DjangoIntegration sentry_sdk.init( integrations=[DjangoIntegration()], send_default_pii=True, - ) # noqa{% endif %}{% if "s3" in cookiecutter.media_storage %} - - # Django Storages - # https://django-storages.readthedocs.io/en/stable/ - - AWS_LOCATION = values.Value("") - - AWS_S3_ENDPOINT_URL = values.Value() - - AWS_S3_FILE_OVERWRITE = values.BooleanValue(False) - - AWS_STORAGE_BUCKET_NAME = values.Value() # noqa{% endif %} + ){% endif %}